update

ansible_local.yml

@@ -112,12 +112,6 @@
           that:
             - "{{ansible_connection in BOX_ANSIBLE_CONNECTIONS}}"
 
-      - name: "we will use sudo and make it a prerequisite"
-        shell: |
-          which sudo || exit 1
-          # "check ansible_python_interpreter"
-          "{{ansible_python_interpreter|default('python3')}}" --version
-
       # required
       tags: always
       check_mode: false
@@ -184,6 +178,20 @@
       check_mode: false
       when: ansible_connection == 'libvirt_qemu'
 
+    - block:
+        
+      # after spinup
+      - name: "we will use sudo and make it a prerequisite"
+        shell: |
+          [ -z "$TMPDIR" ] || [ -d "$TMPDIR" ] || mkdir -p "$TMPDIR"
+          which sudo || exit 1
+          # "check ansible_python_interpreter"
+          "{{ansible_python_interpreter|default('python3')}}" --version
+
+      # required
+      tags: always
+      check_mode: false
+
 #    # required?
 #    tags: always
 #    check_mode: false
@@ -202,10 +210,10 @@
       when:
         - "'proxy' in ROLES"
 
-    - role: ansible-gentoo_install
-      when:
-        # BOX_OS_FAMILY == 'Gentoo' or BOX_GENTOO_FROM_MP != '' ?
-        - ( ansible_connection == 'local' and nbd_disk|default('') != '' ) or (ansible_connection == 'chroot' )
+#    - role: ansible-gentoo_install
+#      when:
+#        # BOX_OS_FAMILY == 'Gentoo' or BOX_GENTOO_FROM_MP != '' ?
+#        - ( ansible_connection == 'local' and nbd_disk|default('') != '' ) or (ansible_connection == 'chroot' )
 
     - role: toxcore
       tags: always

library/#ansible_gentooimgr.py#

@@ -1,241 +0,0 @@
-#!/usr/bin/python3
-
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-import os
-import sys
-import logging
-from argparse import Namespace
-import pathlib
-import traceback
-
-# in the library
-mod_path = ''
-if os.environ.get('PLAY_ANSIBLE_SRC',''):
-    # running from source
-    mod_path = os.environ.get('PLAY_ANSIBLE_SRC','')
-    mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
-    assert os.path.isdir(mod_path), f"parent {mod_path}"
-    assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
-    assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
-    sys.path.append(mod_path)
-else:
-    # in the library
-    mod_path = os.path.dirname(os.path.realpath('__file__'))
-    mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
-    assert os.path.isdir(mod_path), f"parent {mod_path}"
-    assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
-    assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
-    sys.path.append(mod_path)
-try:
-    import gentooimgr
-except Exception as e:
-    sys.stderr.write(f"{mod_path} {sys.path} {traceback.print_exc()}")
-    raise
-import ansible
-
-DOCUMENTATION = rf'''
----
-module: gentooimgr
-
-short_description: Gentoo Image Builder for Cloud and Turnkey ISO installers
-
-
-version_added: "1.0.0"
-
-description:
-* This project enables easy access to building ``systemd`` or ``openrc`` -based images.
-* Performs automatic download AND verification of the linux iso, stage3 tarball and portage.
-* Caches the iso and stage3 .txt files for at most a day before redownloading and rechecking for new files
-* Sane and readable cli commands to build, run and test.
-* Step system to enable user to continue off at the same place if a step fails
-* No heavy packages like rust included ** TODO
-
-options:
-    action:
-        description: The action to be run by the image builder
-        choices:
-            - build
-            - run
-            - status
-            - install
-            - chroot
-            - unchroot
-            - command
-            - shrink
-            - kernel
-        required: true
-        # clean test
-    config:
-        default: cloud.json
-        description: init configuration file or or base.json or cloud.json
-        required: false
-    loglevel:
-        default: {logging.INFO}
-        description: python logging level <= 50, INFO=20
-        required: false
-    threads:
-        default: 1
-        description: Number of threads to use
-        required: false
-    profile:
-        default: openrc
-        description: The init system
-        choices:
-            - openrc
-            - systemd
-        required: false
-    kernel_dir:
-        default: /usr/src/linux
-        description: Where kernel is specified. By default uses the active linux kernel
-        required: false
-    portage:
-        description: Extract the specified portage tarball onto the filesystem
-        required: false
-    stage3:
-        description: Extract the specified stage3 package onto the filesystema
-        required: false
-    action_args:
-        default: []
-        description: Arguments for some of the actions - UNUSED!
-        required: false
-    temporary_dir:
-        description: Path to temporary directory for downloading files (20G)
-        required: false
-    qcow:
-        description: Path to file to serve as the base image
-        required: false
-
-# Specify this value according to your collection
-# in format of namespace.collection.doc_fragment_name
-# extends_documentation_fragment:
-#     - my_namespace.my_collection.my_doc_fragment_name
-
-author:
-    - Your Name (@yourGitHubHandle)
-'''
-
-#[-y DAYS]
-#                  [-d DOWNLOAD_DIR]
-# [-f]
-#                  [--format FORMAT]
-
-EXAMPLES = r'''
-# Pass in a message
-- name: Test with a message
-  my_namespace.my_collection.my_test:
-    name: hello world
-
-# pass in a message and have changed true
-- name: Test with a message and changed output
-  my_namespace.my_collection.my_test:
-    name: hello world
-    new: true
-
-# fail the module
-- name: Test failure of the module
-  my_namespace.my_collection.my_test:
-    name: fail me
-'''
-
-RETURN = r'''
-# These are examples of possible return values, and in general should use other names for return values.
-message:
-    description: The output message that the test module generates.
-    type: str
-    returned: always
-    sample: 'goodbye'
-'''
-
-from ansible.module_utils.basic import AnsibleModule
-
-
-def run_module():
-    # define available arguments/parameters a user can pass to the module
-    #? default config from __file__ ?
-    if mod_path and os.path.isdir(mod_path):
-        def_config = os.path.join(mod_path, 'configs', 'base.json')
-    else:
-        # WARN:
-        def_config = 'base.json'
-    module_args = dict(
-        action=dict(type='str', required=True),
-        loglevel=dict(type='int', required=False, default=logging.INFO),
-        threads=dict(type='int', required=False, default=1),
-        config=dict(type='path', default=def_config, required=True),
-        profile=dict(type='str', required=False),
-        kernel_dir=dict(type='path', required=False),
-        portage=dict(type='path', required=False),
-        stage3=dict(type='path', required=False),
-        temporary_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
-        download_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
-        qcow=dict(type='path', required=False),
-    )
-
-    # seed the result dict in the object
-    # we primarily care about changed and state
-    # changed is if this module effectively modified the target
-    # state will include any data that you want your module to pass back
-    # for consumption, for example, in a subsequent task
-    result = dict(
-        changed=False,
-        original_message='',
-        message=''
-    )
-
-    # the AnsibleModule object will be our abstraction working with Ansible
-    # this includes instantiation, a couple of common attr would be the
-    # args/params passed to the execution, as well as if the module
-    # supports check mode
-    module = AnsibleModule(
-        argument_spec=module_args,
-        supports_check_mode=True
-    )
-
-    # if the user is working with this module in only check mode we do not
-    # want to make any changes to the environment, just return the current
-    # state with no modifications
-    if module.check_mode:
-        module.exit_json(**result)
-
-    # manipulate or modify the state as needed (this is going to be the
-    # part where your module will do what it needs to do)
-    # if module.params.get('thirsty'):
-
-    oargs = Namespace(**module.params)
-    # during the execution of the module, if there is an exception or a
-    # conditional state that effectively causes a failure, run
-    # AnsibleModule.fail_json() to pass in the message and the result
-    result['original_message'] = ""
-    try:
-        from gentooimgr.__main__ import main
-        retval = main(oargs)
-        # should be 0
-        # is stdout already in result? how can it be?
-    except Exception as e:
-        result['message'] = str(e)
-        e = traceback.print_exc()
-        if e: result['original_message'] += f"{e}"
-        module.fail_json(msg='Exception', **result)
-    else:
-        result['message'] = str(retval)
-
-    # use whatever logic you need to determine whether or not this module
-    # made any modifications to your target
-    if dArgs['action'] in ['status', '']:
-        result['changed'] = False
-    else:
-        result['changed'] = True
-
-    # in the event of a successful module execution, you will want to
-    # simple AnsibleModule.exit_json(), passing the key/value results
-    module.exit_json(**result)
-
-
-def main():
-    run_module()
-
-if __name__ == '__main__':
-    main()
-

library/ansible_gentooimgr.py

@@ -223,7 +223,8 @@ def run_module():
 
     # use whatever logic you need to determine whether or not this module
     # made any modifications to your target
-    if dArgs['action'] in ['status', '']:
+    # build run test chroot unchroot status clean kernel shrink
+    if oargs.action in ['status', '']:
         result['changed'] = False
     else:
         result['changed'] = True

roles/ansible-gentoo_install/tasks/#local.yml#

@@ -0,0 +1,183 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+---
+- name: "DEBUG: ansible-gentoo_install local"
+  debug:
+    verbosity: 0
+    msg: "DEBUG: ansible-gentoo_install local BOX_NBD_DEV={{BOX_NBD_DEV}}"
+  check_mode: no
+
+- assert:
+    that:
+      -  "'{{BOX_NBD_DEV}}' != ''"
+  when: ansible_connection in ['local', 'chroot']
+
+- set_fact:
+    AGI_use_local_kernel: true
+  when:
+    - ansible_distribution == 'Gentoo' or BOX_GENTOO_FROM_MP not in ['/', '']
+
+- set_fact:
+    AGI_PROXY_MODE: "{{PROXY_MODE|default('')}}"
+  when:
+     - PROXY_MODE|default('') != ''
+  check_mode: no
+
+- set_fact:
+    AGI_PROXY_MODE: "{{BOX_PROXY_MODE|default('')}}"
+  when:
+     - AGI_PROXY_MODE == ''
+  check_mode: no
+
+- block:
+
+  - name: check for mounted disk
+    shell: |
+      grep '/dev/{{AGI_NBD_DEV}}' /proc/mounts && exit 0
+      ps ax | grep -v grep | \
+        grep "qemu-nbd.*/dev/nbd.*{{BOX_NBD_BASE_QCOW}}" && \
+        echo WARN looks like theres an active nbd mount of \
+        "${BOX_NBD_BASE_QCOW}" && exit 1
+      exit 2
+    failed_when: false
+    changed_when: false
+    register: check_mounted_disk
+    check_mode: no
+
+  - name: partition if disk not mounted
+    fail:
+      msg: "looks like theres an active nbd mount of {{BOX_NBD_BASE_QCOW}}"
+    when:
+      - check_mounted_disk.rc == 1
+    check_mode: no
+
+  - name: partition if disk not mounted or active
+    include: disk.yml
+    when:
+      - check_mounted_disk.rc > 1
+    check_mode: no
+
+  - name: mount root partition
+    mount:
+      name: "{{AGI_NBD_MP}}"
+      src: "{{ AGI_install_disk }}p3"
+      fstype: ext4
+      state: mounted
+    check_mode: false
+
+  - name: create /boot mountpoint
+    file:
+      path: "{{AGI_NBD_MP}}/boot"
+      state: directory
+    check_mode: false
+
+  - name: mount boot partition
+    mount:
+      name: "{{AGI_NBD_MP}}/boot"
+      src: "{{ AGI_install_disk }}p1"
+      fstype: ext2
+      state: mounted
+    check_mode: false
+
+  - include: tarball.yml
+  - include: copy.yml
+    when: AGI_use_local_kernel
+
+  - name: mount distfiles
+    delegate_to: localhost
+    shell: |
+      [ -d "{{MOUNT_GENTOO_DISTFILES_ARCHIVES}}" ] || exit 1
+      grep {{MOUNT_GENTOO_DISTFILES_ARCHIVES}} /proc/mounts && exit 0
+      [ -d {{AGI_NBD_MP}}/usr/portage/ ] || exit 0
+      [ -d {{AGI_NBD_MP}}/usr/portage/distfiles ] || mkdir {{AGI_NBD_MP}}/usr/portage/distfiles
+      mount --bind {{MOUNT_GENTOO_DISTFILES_ARCHIVES}} {{AGI_NBD_MP}}/usr/portage/distfiles
+    when:
+      - "MOUNT_GENTOO_DISTFILES_ARCHIVES != ''"
+      - "AGI_NBD_MP != ''"
+
+  - include: chroot.yml
+
+  delegate_to: localhost
+  when: ansible_connection in ['chroot', 'local'] # libvirt?
+
+- block:
+
+  - name: check chroot wrapper installed
+    shell: |
+      [ -x /var/tmp/chroot_wrapper.sh ] || exit 1
+      df /mnt/gentoo || exit 2
+      /var/tmp/chroot_wrapper.sh /bin/df | grep /mnt/gentoo && exit 4
+      exit 0
+    register: chroot_out
+    check_mode: false
+
+  - name: enable chroot wrapper
+    set_fact:
+      ansible_shell_executable: /var/tmp/chroot_wrapper.sh
+      old_ansible_python_interpreter: "{{ansible_python_interpreter}}"
+      ansible_python_interpreter: "/usr/bin/python3"
+    check_mode: false
+
+  when: ansible_connection in ['local']
+
+- block:
+
+  - include: portage.yml
+  - include: misc.yml
+
+  - include: network.yml
+
+  - include: kernel.yml
+    when: not AGI_use_local_kernel
+
+  - include: bootloader.yml
+  - include: daemons.yml
+
+  #   - include: finish.yml
+
+  check_mode: false
+  when:
+    - "ansible_connection in ['chroot'] or (ansible_connection in ['local'] or  and chroot_out.rc|default(1) == 0)"
+  rescue:
+    - debug:
+        msg: "ERROR: error during chroot execution"
+
+- name: disable chroot wrapper
+  set_fact:
+    ansible_shell_executable: /bin/sh
+    ansible_python_interpreter: "{{old_ansible_python_interpreter}}"
+  when:
+    - "ansible_connection in ['local'] and chroot_out.rc|default(1) == 0"
+  check_mode: false
+
+- name: unmount filesystems
+  mount:
+    name: "{{AGI_NBD_MP}}/{{ item }}"
+    state: unmounted
+  with_items:
+    - proc
+    - sys
+    - dev/pts
+    - dev/shm
+    - dev
+    - boot
+    - ''
+  loop_control:
+    label: "{{AGI_NBD_MP}}/{{ item }}"
+  when:
+    - "ansible_connection in ['local'] and chroot_out.rc|default(1) == 0"
+    - false # leave it mounted for testing
+
+- name: dismount any other mounts
+  shell: |
+    if [ -z "{{MOUNT_GENTOO_DISTFILES_ARCHIVES}}" ] && \
+         [ -d "{{MOUNT_GENTOO_DISTFILES_ARCHIVES}}" ] && \
+         grep {{MOUNT_GENTOO_DISTFILES_ARCHIVES}} /proc/mounts ; then
+           umount  {{MOUNT_GENTOO_DISTFILES_ARCHIVES}}
+      fi
+    df -a | grep "{{AGI_NBD_MP}}" | sed -e 's/.* //' | tac | while read elt;do
+        umount $elt
+        done
+    base_chroot_unbind.bash "{{AGI_NBD_MP}}"
+  when:
+    - "ansible_connection in ['chroot'] or chroot_out.rc|default(1) == 0"
+    - false # leave it mounted for testing

roles/ansible-gentoo_install/tasks/copy.yml

@@ -16,15 +16,6 @@
     [ -d "{{AGI_GENTOO_FROM_MP}}" ] || exit 5
   check_mode: false
 
-- name: install to mp from source
-  delegate_to: localhost
-  shell: |
-    cd {{AGI_GENTOO_FROM_MP}}/usr/src/linux || exit 1
-    [ -d "{{AGI_NBD_MP}}/lib/modules" ] || mkdir "{{AGI_NBD_MP}}/lib/modules"
-    make INSTALL_PATH={{AGI_NBD_MP}}/boot install || exit 4
-    make INSTALL_MOD_PATH={{AGI_NBD_MP}} modules_install || exit 5
-  when: AGI_use_local_kernel
-
 - name: resolve kernel symlink
   shell: |
     [ -h {{AGI_GENTOO_FROM_MP}}/usr/src/linux ] && \
@@ -34,16 +25,26 @@
   register: kernel_out
   check_mode: false
 
+- name: install to mp from source
+  delegate_to: localhost
+  shell: |
+    cd {{AGI_GENTOO_FROM_MP}}/usr/src/linux || exit 1
+    [ -d "{{AGI_NBD_MP}}/lib/modules" ] || mkdir "{{AGI_NBD_MP}}/lib/modules"
+    make INSTALL_PATH={{AGI_NBD_MP}}/boot install || exit 4
+    make INSTALL_MOD_PATH={{AGI_NBD_MP}} modules_install || exit 5
+  args:
+    creates: "{{AGI_NBD_MP}}/lib/modules/{{kernel_out.stdout}}"
+  when: AGI_use_local_kernel
+
 - name: copy kernel sources
   copy:
     src:  "{{AGI_GENTOO_FROM_MP}}/usr/src/{{kernel_out.stdout}}"
     dest: "{{AGI_NBD_MP}}/usr/src"
     remote_src: no
-    creates: "{{AGI_NBD_MP}}/usr/src/"
+    creates: "{{AGI_NBD_MP}}/usr/src/{{kernel_out.stdout}}"
   when:
     - kernel_out.rc|default(1) == 0
     - AGI_use_local_kernel
-    - false # dunno where it went to
 
 - name: resolve kver
   shell: |
@@ -102,13 +103,14 @@
 - name: make directories
   shell: |
     cd {{AGI_GENTOO_FROM_MP}} || exit 1
-    for dir in {{AGI_bootstrap_dirs}}; do
+    for dir in {{' '.join(AGI_bootstrap_dirs)}}; do
         [ -d "{{AGI_NBD_MP}}/$dir" ] && continue
         mkdir -p "{{AGI_NBD_MP}}/$dir"
     done
-    for file in {{AGI_bootstrap_files}}; do
+    for file in {{' '.join(AGI_bootstrap_files)}}; do
         [ -f "{{AGI_NBD_MP}}/$file" ] && continue
         cp -np "$file" "{{AGI_NBD_MP}}/$file"
     done
+  ignore_errors: false
 
 # dracut

roles/ansible-gentoo_install/tasks/disk.yml

@@ -79,6 +79,14 @@
     check_mode: false
     when: not ansible_check_mode
 
+  - name: label partitions
+    shell: |
+      partprobe
+      e2label  {{ AGI_install_disk }}p3 root
+      e2label  {{ AGI_install_disk }}p1 boot
+      mkswap -L swap  "{{ AGI_install_disk }}p2"
+      sync
+      
   when: false
 
 - block:
@@ -96,13 +104,11 @@
       {{ AGI_install_disk }}p2 : start=      821248, size=     4096000, type=82
       {{ AGI_install_disk }}p3 : start=     4917248, size=    37025792, type=83
       EOF
-      mke2fs {{ AGI_install_disk }}p1
-      mke2fs {{ AGI_install_disk }}p3
+      partprobe
+      mke2fs -L boot {{ AGI_install_disk }}p1
+      mke2fs -L root {{ AGI_install_disk }}p3
+      mkswap -L swap  "{{ AGI_install_disk }}p2"
+      sync
       
   when: true
 
-- name: label partitions
-  shell: |
-    e2label  {{ AGI_install_disk }}p3 root
-    e2label  {{ AGI_install_disk }}p1 boot
-    mkswap -L swap  "{{ AGI_install_disk }}p2"

roles/ansible-gentoo_install/tasks/local.yml

@@ -78,6 +78,10 @@
       state: mounted
     check_mode: false
 
+  - include: tarball.yml
+  - include: copy.yml
+    when: AGI_use_local_kernel
+
   - name: mount distfiles
     delegate_to: localhost
     shell: |
@@ -89,11 +93,7 @@
     when:
       - "MOUNT_GENTOO_DISTFILES_ARCHIVES != ''"
       - "AGI_NBD_MP != ''"
-      - false # let the tester take care of this
 
-  - include: tarball.yml
-  - include: copy.yml
-    when: AGI_use_local_kernel
   - include: chroot.yml
 
   delegate_to: localhost
@@ -105,7 +105,7 @@
     shell: |
       [ -x /var/tmp/chroot_wrapper.sh ] || exit 1
       df /mnt/gentoo || exit 2
-      /var/tmp/chroot_wrapper.sh df | grep /mnt/gentoo && exit 4
+      /var/tmp/chroot_wrapper.sh /bin/df | grep /mnt/gentoo && exit 4
       exit 0
     register: chroot_out
     check_mode: false
@@ -136,10 +136,10 @@
 
   check_mode: false
   when:
-    - "ansible_connection in ['chroot'] or chroot_out.rc|default(1) == 0"
+    - "ansible_connection in ['chroot'] or (ansible_connection in ['local'] or  and chroot_out.rc|default(1) == 0)"
   rescue:
     - debug:
-        msg: "ERROR: "
+        msg: "ERROR: error during chroot execution"
 
 - name: disable chroot wrapper
   set_fact:
@@ -177,6 +177,7 @@
     df -a | grep "{{AGI_NBD_MP}}" | sed -e 's/.* //' | tac | while read elt;do
         umount $elt
         done
+    base_chroot_unbind.bash "{{AGI_NBD_MP}}"
   when:
     - "ansible_connection in ['chroot'] or chroot_out.rc|default(1) == 0"
     - false # leave it mounted for testing

roles/toxcore/tasks/main.yml

@@ -114,7 +114,7 @@
     df | grep /boot || mount /boot || exit 2
     [ -d /boot/grub ] || exit 3
     [ -f /boot/grub/grub.cfg ] || exit 4
-    [ -f /boot/grub/grub.cfg ] && cp -p /boot/grub/grub.cfg /boot/grub/grub.cfg.dst
+    [ -f /boot/grub/grub.cfg.dst ] || cp -p /boot/grub/grub.cfg /boot/grub/grub.cfg.dst
     sed -e 's@ ro *$@ '"$LINE"' ro@' -i /boot/grub/grub.cfg
   ignore_errors: true
 
@@ -284,7 +284,19 @@
     [ '{{BOX_NBD_OVERLAY_DIR}}' != '' ] || exit 2
     [ '{{BOX_NBD_BASE_PUBKEY}}' != '' ] || exit 3
     [ '{{BOX_NBD_BASE_QCOW}}' != '' ] || exit 4
-    [ -f '{{BOX_NBD_BASE_QCOW}}' ] || exit 5
+    [ '{{BOX_NBD_OVERLAY_RAM}}' != '' ] || exit 5
+    [ '{{BOX_NBD_OVERLAY_BR}}' != '' ] || exit 6
+    [ '{{BOX_NBD_OVERLAY_GB}}' != '' ] || exit 7
+    [ '{{BOX_NBD_OVERLAY_CPUS}}' != '' ] || exit 8
+    [ '{{BOX_NBD_OVERLAY_PASS}}' != '' ] || exit 9
+    if [ ! -f '{{BOX_NBD_BASE_QCOW}}' ] ; then
+        echo WARN: '{{BOX_NBD_BASE_QCOW}}' not built yet - skipping
+        exit 0
+    fi
+    if [ -z '{{BOX_NBD_BASE_PASS}}' ] ; then
+        echo WARN: 'BOX_NBD_BASE_PASS' empty
+        exit 10
+    fi
     toxcore_create-vm.bash \
              -n {{BOX_NBD_OVERLAY_NAME}} \
              -k {{BOX_NBD_BASE_PUBKEY}} \
@@ -293,8 +305,8 @@
              -c {{BOX_NBD_OVERLAY_CPUS}} \
              -r {{BOX_NBD_OVERLAY_RAM}} \
              -d {{BOX_NBD_OVERLAY_DIR}} \
-             -b virbr0 \
-             -p gentoo \
+             -b {{BOX_NBD_OVERLAY_BR}}  \
+             -p {{BOX_NBD_OVERLAY_PASS}} \
              -o gentoo
   ignore_errors: true
   when:

roles/toxcore/vars/Gentoo2.yml

@@ -40,7 +40,6 @@ toxcore_pkgs_inst:
   - dev-python/requests
   - gnome-base/dconf
   - net-libs/gtk-vnc
-  - net-misc/spice-gtk
   - sys-apps/dbus
   - x11-libs/gtk+
   - x11-libs/gtksourceview

src/ansible_gentooimgr/gentooimgr/#install.py#

@@ -0,0 +1,360 @@
+"""Configure a Gentoo guest with cloud image settings
+
+This step keeps track of how far it's gotten, so re-running this command
+will continue on if an error was to occur, unless --start-over flag is given.
+"""
+
+import os
+import sys
+import shutil
+import configparser
+from subprocess import Popen, PIPE
+import logging
+import traceback
+
+import gentooimgr.config
+import gentooimgr.configs
+import gentooimgr.common
+import gentooimgr.chroot
+import gentooimgr.kernel
+from gentooimgr import LOG
+from gentooimgr import HERE
+
+from gentooimgr.configs import *
+
+FILES_DIR = os.path.join(HERE, "..")
+
+def step1_diskprep(args, cfg):
+    LOG.info("\t:: Step 1: Disk Partitioning")
+    # http://rainbow.chard.org/2013/01/30/how-to-align-partitions-for-best-performance-using-parted/
+    # http://honglus.blogspot.com/2013/06/script-to-automatically-partition-new.html
+    cmds = [
+            ['parted', '-s', f'{cfg.get("disk")}', 'mklabel', 'msdos'],
+            ['parted', '-s', f'{cfg.get("disk")}', 'mkpart', 'primary', '2048s', '100%'],
+            ['partprobe'],
+            ['mkfs.ext4', '-FF', f'{cfg.get("disk")}{cfg.get("partition", 1)}']
+    ]
+    for c in cmds:
+        proc = Popen(c, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+
+    completestep(1, "diskprep")
+
+def step2_mount(args, cfg):
+    LOG.info(f'\t:: Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}')
+    proc = Popen(["mount", f'{cfg.get("disk")}{cfg.get("partition")}', cfg.get("mountpoint")])
+    proc.communicate()
+    completestep(2, "mount")
+
+def step3_stage3(args, cfg):
+    LOG.info(f'\t:: Step 3: Stage3 Tarball')
+
+    stage3 = cfg.get("stage3") or args.stage3  # FIXME: auto detect stage3 images in mountpoint and add here
+    if not stage3:
+        stage3 = gentooimgr.common.stage3_from_dir(FILES_DIR)
+
+
+    proc = Popen(["tar", "xpf", os.path.abspath(stage3), "--xattrs-include='*.*'", "--numeric-owner", "-C",
+                  f'{cfg.get("mountpoint")}'])
+    proc.communicate()
+    completestep(3, "stage3")
+
+def step4_binds(args, cfg):
+    LOG.info(f'\t:: Step 4: Binding Filesystems')
+    gentooimgr.chroot.bind(verbose=False)
+    completestep(4, "binds")
+
+def step5_portage(args, cfg):
+    LOG.info(f'\t:: Step 5: Portage')
+    portage = cfg.get("portage") or args.portage
+    if not portage:
+        portage = gentooimgr.common.portage_from_dir(FILES_DIR)
+    proc = Popen(["tar", "xpf", portage, "-C", f"{cfg.get('mountpoint')}/usr/"])
+    proc.communicate()
+    # Edit portage
+    portage_env = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'env')
+    os.makedirs(portage_env, exist_ok=True)
+    with open(os.path.join(portage_env, 'singlejob.conf'), 'w') as f:
+        f.write('MAKEOPTS="-j1"\n')
+
+    env_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.env')
+    with open(env_path, 'w') as f:
+        f.write("app-portage/eix singlejob.conf\ndev-util/maturin singlejob.conf\ndev-util/cmake singlejob.conf")
+
+    completestep(5, "portage")
+
+def step6_licenses(args, cfg):
+    LOG.info(f'\t:: Step 6: Licenses')
+    license_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.license')
+    os.makedirs(license_path, exist_ok=True)
+    for f, licenses in cfg.get("licensefiles", {}).items():
+        with open(os.path.join(license_path, f), 'w') as f:
+            f.write('\n'.join(licenses))
+    completestep(6, "license")
+
+def step7_repos(args, cfg):
+    LOG.info(f'\t:: Step 7: Repo Configuration')
+    repo_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'repos.conf')
+    os.makedirs(repo_path, exist_ok=True)
+    # Copy from template
+    repo_file = os.path.join(repo_path, 'gentoo.conf')
+    shutil.copyfile(
+        os.path.join(cfg.get("mountpoint"), 'usr', 'share', 'portage', 'config', 'repos.conf'),
+        repo_file)
+    # Regex replace lines
+    cp = configparser.ConfigParser()
+    for repofile, data in cfg.get("repos", {}).items():
+        cp.read(cfg.get("mountpoint") + repofile)  # repofile should be absolute path, do not use os.path.join.
+        for section, d in data.items():
+            if section in cp:
+                for key, val in d.items():
+                    # Replace everything after the key with contents of value.
+                    # Sed is simpler than using regex for this purpose.
+                    cp.set(section, key, val)
+            else:
+                sys.stderr.write(f"\tWW No section {section} in {repofile}\n")
+
+        cp.write(open(cfg.get("mountpoint") + repofile, 'w'))
+
+    completestep(7, "repos")
+
+def step8_resolv(args, cfg):
+    LOG.info(f'\t:: Step 8: Resolv')
+    proc = Popen(["cp", "--dereference", "/etc/resolv.conf", os.path.join(cfg.get("mountpoint"), 'etc')])
+    proc.communicate()
+    # Copy all step files and python module to new chroot
+    os.system(f"cp /tmp/*.step {cfg.get('mountpoint')}/tmp")
+    os.system(f"cp -r . {cfg.get('mountpoint')}/mnt/")
+    completestep(8, "resolv")
+
+def step9_sync(args, cfg):
+    LOG.info(f"\t:: Step 9: sync")
+    LOG.info("\t\t:: Entering chroot")
+    os.chroot(cfg.get("mountpoint"))
+    os.chdir(os.sep)
+    os.system("source /etc/profile")
+    proc = Popen(["emerge", "--sync", "--quiet"])
+    proc.communicate()
+    LOG.info("\t\t:: Emerging base")
+    proc = Popen(["emerge", "--update", "--deep", "--newuse", "--keep-going", "@world"])
+    proc.communicate()
+    completestep(9, "sync")
+
+def step10_emerge_pkgs(args, cfg):
+    LOG.info(f"\t:: Step 10: emerge pkgs")
+    packages = cfg.get("packages", {})
+    for oneshot_up in packages.get("oneshots", []):
+        proc = Popen(["emerge", "--oneshot", "--update", oneshot_up])
+        proc.communicate()
+
+    for single in packages.get("singles", []):
+        proc = Popen(["emerge", "-j1", single])
+        proc.communicate()
+
+    LOG.info(f"KERNEL PACKAGES {packages.get('kernel')}")
+    if packages.get("kernel", []):
+        cmd = ["emerge", "-j", str(args.threads)] + packages.get("kernel", [])
+        proc = Popen(cmd)
+        proc.communicate()
+
+    cmd = ["emerge", "-j", str(args.threads), "--keep-going"]
+    cmd += packages.get("keepgoing", [])
+    proc = Popen(cmd)
+    proc.communicate()
+
+    cmd = ["emerge", "-j", str(args.threads)]
+    cmd += packages.get("base", [])
+    cmd += packages.get("additional", [])
+    cmd += packages.get("bootloader", [])
+    LOG.info(cmd)
+    proc = Popen(cmd)
+    proc.communicate()
+    completestep(10, "pkgs")
+
+def step11_kernel(args, cfg):
+    # at this point, genkernel will be installed
+    LOG.info(f"\t:: Step 11: kernel")
+    proc = Popen(["eselect", "kernel", "set", "1"])
+    proc.communicate()
+    if not args.kernel_dist:
+        os.chdir(args.kernel_dir)
+        threads = str(gentooimgr.config.THREADS)
+        gentooimgr.kernel.build_kernel(args, cfg)
+
+    completestep(11, "kernel")
+
+def step12_grub(args, cfg):
+    LOG.info(f"\t:: Step 12: kernel")
+    proc = Popen(["grub-install", cfg.get('disk')])
+    proc.communicate()
+    code = proc.returncode
+    if code != 0:
+        sys.stderr.write(f"Failed to install grub on {cfg.get('disk')}\n")
+        sys.exit(code)
+
+    with open("/etc/default/grub", 'w') as f:
+        f.write(f"{gentooimgr.kernel.GRUB_CFG}")
+
+    proc = Popen(["grub-mkconfig", "-o", "/boot/grub/grub.cfg"])
+    proc.communicate()
+    completestep(12, "grub")
+
+def step13_serial(args, cfg):
+    LOG.info(f"\t:: Step 13: Serial")
+    os.system("sed -i 's/^#s0:/s0:/g' /etc/inittab")
+    os.system("sed -i 's/^#s1:/s1:/g' /etc/inittab")
+    completestep(13, "serial")
+
+def step14_services(args, cfg):
+    LOG.info(f"\t:: Step 14: Services")
+    for service in ["acpid", "syslog-ng", "cronie", "sshd", "cloud-init-local", "cloud-init", "cloud-config",
+                    "cloud-final", "ntpd", "nfsclient"]:
+        if args.profile == "systemd":
+            proc = Popen(["systemctl", "enable", service])
+        else:
+            proc = Popen(["rc-update", "add", service, "default"])
+        proc.communicate()
+
+    completestep(14, "services")
+
+def step15_ethnaming(args, cfg):
+    LOG.info(f"\t:: Step 15: Eth Naming")
+    completestep(15, "networking")
+
+def step16_sysconfig(args, cfg):
+    LOG.info(f"\t:: Step 16: Sysconfig")
+    with open("/etc/timezone", "w") as f:
+        f.write("UTC")
+    proc = Popen(["emerge", "--config", "sys-libs/timezone-data"])
+    proc.communicate()
+    with open("/etc/locale.gen", "a") as f:
+        f.write("en_US.UTF-8 UTF-8\nen_US ISO-8859-1\n")
+    proc = Popen(["locale-gen"])
+    proc.communicate()
+    proc = Popen(["eselect", "locale", "set", "en_US.utf8"])
+    proc.communicate()
+    proc = Popen(["env-update"])
+    proc.communicate()
+    with open('/etc/sysctl.d/swappiness.conf', 'w') as f:
+        f.write("vm.swappiness = 0\n")
+
+    modloadpath = os.path.join(os.sep, 'etc', 'modules-load.d')
+    os.makedirs(modloadpath, exist_ok=True)
+    with open(os.path.join(modloadpath, 'cloud-modules.conf'), 'w') as f:
+        f.write('\n'.join(gentooimgr.config.CLOUD_MODULES))
+
+    cloudcfg = os.path.join(os.sep, 'etc', 'cloud')
+    if not os.path.exists(cloudcfg):
+        os.makedirs(cloudcfg, exist_ok=True)
+        os.makedirs(os.path.join(cloudcfg, 'templates'), exist_ok=True)
+    with open(os.path.join(cloudcfg, 'cloud.cfg'), 'w') as cfg:
+        cfg.write(f"{CLOUD_YAML}")
+
+    os.chmod(os.path.join(cloudcfg, "cloud.cfg"), 0o644)
+
+    with open(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 'w') as tmpl:
+        tmpl.write(f"{HOST_TMPL}") # FIXME:
+
+    os.chmod(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 0o644)
+
+    proc = Popen("sed -i 's/domain_name\,\ domain_search\,\ host_name/domain_search/g' /etc/dhcpcd.conf", shell=True)
+    proc.communicate()
+
+    hostname = os.path.join(os.sep, 'etc', 'conf.d', 'hostname')
+    with open(hostname, 'w') as f:
+        f.write(f"{HOSTNAME}\n")
+
+    os.chmod(hostname, 0o644)
+
+    proc = Popen(["eix-update"])
+    proc.communicate()
+
+    os.remove(os.path.join(os.sep, 'etc', 'resolv.conf'))
+
+    completestep(16, "sysconfig")
+
+def step17_fstab(args, cfg):
+    LOG.info(f"\t:: Step 17: fstab")
+    with open(os.path.join(os.sep, 'etc', 'fstab'), 'a') as fstab:
+        fstab.write(f"{cfg.get('disk')}\t/\text4\tdefaults,noatime\t0 1\n")
+
+    completestep(17, "fstab")
+
+def completestep(step, stepname, prefix='/tmp'):
+    with open(os.path.join(prefix, f"{step}.step"), 'w') as f:
+        f.write("done.")  # text in this file is not currently used.
+
+
+def getlaststep(prefix='/tmp'):
+    i = 1
+    found = False
+    while not found:
+        if os.path.exists(f"{i}.step"):
+            i += 1
+        else:
+            found = True
+
+    return i
+
+
+def stepdone(step, prefix='/tmp'):
+    return os.path.exists(os.path.join(prefix, f"{step}.step"))
+
+def configure(args, config: dict):
+    # Load configuration
+    if not os.path.exists(gentooimgr.config.GENTOO_MOUNT):
+        if not args.force:
+            # We aren't in a gentoo live cd are we?
+            sys.stderr.write("Your system doesn't look like a gentoo live cd, exiting for safety.\n"
+                "If you want to continue, use --force option and re-run `python -m gentooimgr install` with your configuration\n")
+            sys.exit(1)
+
+        else:
+            # Assume we are root as per live cd, otherwise user should run this as root as a secondary confirmation
+            os.makedirs(gentooimgr.config.GENTOO_MOUNT)
+    # disk prep
+    cfg = config
+    # must be root for linux-6.1.52-pentoo/certs/signing_key.pem
+    if not stepdone(1): step1_diskprep(args, cfg)
+    # mount root
+    if not stepdone(2): step2_mount(args, cfg)
+    # extract stage
+    if not stepdone(3): step3_stage3(args, cfg)
+    # mount binds
+    if not stepdone(4): step4_binds(args, cfg)
+    # extract portage
+    if not stepdone(5): step5_portage(args, cfg)
+    # Set licenses
+    if not stepdone(6): step6_licenses(args, cfg)
+    # repos.conf
+    if not stepdone(7): step7_repos(args, cfg)
+    # portage env files and resolv.conf
+    if not stepdone(8): step8_resolv(args, cfg)
+    # emerge --sync
+    if not stepdone(9): step9_sync(args, cfg)
+    # bindist
+    if not stepdone(10): step10_emerge_pkgs(args, cfg)
+    # emerge packages
+    # configure & emerge kernel (use cloud configuration too)
+    if not stepdone(11): step11_kernel(args, cfg)
+    # grub
+    if not stepdone(12): step12_grub(args, cfg)
+    # enable serial console
+    if not stepdone(13): step13_serial(args, cfg)
+    # services
+    if not stepdone(14): step14_services(args, cfg)
+    # eth0 naming
+    # timezone
+    if not stepdone(15): step15_ethnaming(args, cfg)
+    # locale
+    # set some sysctl things
+    # set some dhcp things
+    # hostname
+    if not stepdone(16): step16_sysconfig(args, cfg)
+    # fstab
+    if not stepdone(17): step17_fstab(args, cfg)
+    # copy cloud cfg?
+    gentooimgr.chroot.unbind()
+    # Finish install processes like emaint and eix-update and news read
+
+

src/ansible_gentooimgr/gentooimgr/.#install.py

@@ -0,0 +1 @@
+root@pentoo.152064:1703733868

src/ansible_gentooimgr/gentooimgr/__main__.py

@@ -165,7 +165,7 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     assert args.loglevel < 59
-    if coloredlogs:
+    if coloredlogs is not None:
         # https://pypi.org/project/coloredlogs/
         coloredlogs.install(level=args.loglevel,
                         logger=LOG,

src/ansible_gentooimgr/gentooimgr/builder.py

@@ -8,7 +8,7 @@ import gentooimgr.qemu as qemu
 import gentooimgr.common
 import requests
 
-def build(args: argparse.Namespace, config: dict) -> None:
+def build(args: argparse.Namespace, config: dict) -> str:
     LOG.info(": build")
 
     iso = config.get("iso") or download.download(args)