Merge pull request #1247 from moreati/issue-1118-ci_lib

CI: ci_lib cleanup

.ci/ansible_tests.py

@@ -12,9 +12,8 @@
 import ci_lib
 
 
-TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates')
-TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
-HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
+TMP = ci_lib.TempDir(prefix='mitogen_ci_ansible')
+TMP_HOSTS_DIR = os.path.join(TMP.path, 'hosts')
 
 
 def pause_if_interactive():
@@ -40,13 +39,13 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('job_setup'):
-    os.chdir(TESTS_DIR)
-    os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 8))
+    os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
+    os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
 
-    ci_lib.run("mkdir %s", HOSTS_DIR)
-    for path in glob.glob(TESTS_DIR + '/hosts/*'):
+    os.mkdir(TMP_HOSTS_DIR)
+    for path in glob.glob(os.path.join(ci_lib.ANSIBLE_TESTS_HOSTS_DIR, '*')):
         if not path.endswith('default.hosts'):
-            ci_lib.run("ln -s %s %s", path, HOSTS_DIR)
+            os.symlink(path, os.path.join(TMP_HOSTS_DIR, os.path.basename(path)))
 
     distros = collections.defaultdict(list)
     families = collections.defaultdict(list)
@@ -55,12 +54,14 @@ def pause_if_interactive():
         families[container['family']].append(container['name'])
 
     jinja_env = jinja2.Environment(
-        loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR),
+        loader=jinja2.FileSystemLoader(
+            searchpath=ci_lib.ANSIBLE_TESTS_TEMPLATES_DIR,
+        ),
         lstrip_blocks=True,  # Remove spaces and tabs from before a block
         trim_blocks=True,  # Remove first newline after a block
     )
     inventory_template = jinja_env.get_template('test-targets.j2')
-    inventory_path = os.path.join(HOSTS_DIR, 'target')
+    inventory_path = os.path.join(TMP_HOSTS_DIR, 'test-targets.ini')
 
     with open(inventory_path, 'w') as fp:
         fp.write(inventory_template.render(
@@ -71,16 +72,12 @@ def pause_if_interactive():
 
     ci_lib.dump_file(inventory_path)
 
-    if not ci_lib.exists_in_path('sshpass'):
-        ci_lib.run("sudo apt-get update")
-        ci_lib.run("sudo apt-get install -y sshpass")
-
-
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
     try:
         ci_lib.run('./run_ansible_playbook.py %s -i "%s" %s',
-            playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
+            playbook, TMP_HOSTS_DIR, ' '.join(sys.argv[1:]),
+        )
     except:
         pause_if_interactive()
         raise

.ci/ci_lib.py

@@ -28,14 +28,20 @@
 )
 
 
+GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+ANSIBLE_TESTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible')
+ANSIBLE_TESTS_HOSTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible/hosts')
+ANSIBLE_TESTS_TEMPLATES_DIR = os.path.join(GIT_ROOT, 'tests/ansible/templates')
 DISTRO_SPECS = os.environ.get(
     'MITOGEN_TEST_DISTRO_SPECS',
     'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004',
 )
+IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep')
 IMAGE_TEMPLATE = os.environ.get(
     'MITOGEN_TEST_IMAGE_TEMPLATE',
     'public.ecr.aws/n5z0e8q9/%(distro)s-test',
 )
+TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key')
 
 
 _print = print
@@ -56,19 +62,11 @@ def _have_cmd(args):
         if exc.errno == errno.ENOENT:
             return False
         raise
-    except subprocess.CallProcessError:
+    except subprocess.CalledProcessError:
         return False
     return True
 
 
-def have_apt():
-    return _have_cmd(['apt', '--help'])
-
-
-def have_brew():
-    return _have_cmd(['brew', 'help'])
-
-
 def have_docker():
     return _have_cmd(['docker', 'info'])
 
@@ -185,8 +183,8 @@ def exists_in_path(progname):
 
 
 class TempDir(object):
-    def __init__(self):
-        self.path = tempfile.mkdtemp(prefix='mitogen_ci_lib')
+    def __init__(self, prefix='mitogen_ci_lib'):
+        self.path = tempfile.mkdtemp(prefix=prefix)
         atexit.register(self.destroy)
 
     def destroy(self, rmtree=shutil.rmtree):
@@ -199,20 +197,6 @@ def __enter__(self): pass
     def __exit__(self, _1, _2, _3): pass
 
 
-GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-TMP = TempDir().path
-
-
-# We copy this out of the way to avoid random stuff modifying perms in the Git
-# tree (like git pull).
-src_key_file = os.path.join(GIT_ROOT,
-    'tests/data/docker/mitogen__has_sudo_pubkey.key')
-key_file = os.path.join(TMP,
-    'mitogen__has_sudo_pubkey.key')
-shutil.copyfile(src_key_file, key_file)
-os.chmod(key_file, int('0600', 8))
-
-
 os.environ['PYTHONDONTWRITEBYTECODE'] = 'x'
 os.environ['PYTHONPATH'] = '%s:%s' % (
     os.environ.get('PYTHONPATH', ''),

.ci/debops_common_tests.py

@@ -6,7 +6,8 @@
 import ci_lib
 
 
-project_dir = os.path.join(ci_lib.TMP, 'project')
+TMP = ci_lib.TempDir(prefix='mitogen_ci_debops')
+project_dir = os.path.join(TMP.path, 'project')
 vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
 inventory_path = 'ansible/inventory/hosts'
 docker_hostname = ci_lib.get_docker_hostname()
@@ -22,6 +23,7 @@
 
 
 with ci_lib.Fold('job_setup'):
+    os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
     ci_lib.run('debops-init %s', project_dir)
     os.chdir(project_dir)
 
@@ -45,7 +47,7 @@
             "\n"
             # Speed up slow DH generation.
             "dhparam__bits: ['128', '64']\n"
-            % (ci_lib.key_file,)
+            % (ci_lib.TESTS_SSH_PRIVATE_KEY_FILE,)
         )
 
     with open(inventory_path, 'a') as fp:

.ci/localhost_ansible_tests.py

@@ -10,19 +10,13 @@
 import ci_lib
 
 
-TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
-IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep')
-HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts')
-KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key')
-
-
 with ci_lib.Fold('unit_tests'):
     os.environ['SKIP_MITOGEN'] = '1'
     ci_lib.run('./run_tests -v')
 
 
 with ci_lib.Fold('job_setup'):
-    os.chmod(KEY_PATH, int('0600', 8))
+    os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
     # NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n", 
     # there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually
     if not ci_lib.exists_in_path('sshpass'):
@@ -51,11 +45,11 @@
         subprocess.check_call('sudo chmod 700 ~root/.ssh', shell=True)
         subprocess.check_call('sudo chmod 600 ~root/.ssh/authorized_keys', shell=True)
 
-    os.chdir(IMAGE_PREP_DIR)
+    os.chdir(ci_lib.IMAGE_PREP_DIR)
     ci_lib.run("ansible-playbook -c local -i localhost, macos_localhost.yml")
 
     if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
-        os.chdir(IMAGE_PREP_DIR)
+        os.chdir(ci_lib.IMAGE_PREP_DIR)
         ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
 
     cmd = ';'.join([
@@ -80,7 +74,7 @@
 
 
 with ci_lib.Fold('ansible'):
-    os.chdir(TESTS_DIR)
+    os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
     ci_lib.run('./run_ansible_playbook.py %s %s',
         playbook, ' '.join(sys.argv[1:]))

.github/workflows/tests.yml

@@ -96,7 +96,7 @@ jobs:
           set -o errexit -o nounset -o pipefail
 
           sudo apt-get update
-          sudo apt-get install -y python2-dev python3-pip virtualenv
+          sudo apt-get install -y python2-dev python3-pip sshpass virtualenv
       - name: Show Python versions
         run: |
           set -o errexit -o nounset -o pipefail

docs/changelog.rst

@@ -24,6 +24,8 @@ In progress (unreleased)
 * :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes
 * :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6
 * :gh:issue:`1118` CI: Statically specify test usernames and group names
+* :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir
+* :gh:issue:`1118` CI: Don't share temporary directory between test groupings
 
 
 v0.3.22 (2025-02-04)