From 1fd42af98dac79011496a7956a8cf592b596e2cf Mon Sep 17 00:00:00 2001 From: Mike Crute Date: Thu, 21 May 2020 10:45:05 -0700 Subject: Refactor resolve-profile script This is paving the way for identity broker improvements for opt-in regions. The output is functionally identical between the two scripts modulo the svcs change. Hopefully this makes the transformation process a little more clear. --- profiles/base/1 | 46 ++++--- scripts/resolve-profile.py.in | 273 +++++++++++++++++++----------------------- 2 files changed, 152 insertions(+), 167 deletions(-) diff --git a/profiles/base/1 b/profiles/base/1 index fdda3f0..18decd1 100644 --- a/profiles/base/1 +++ b/profiles/base/1 @@ -56,25 +56,33 @@ pkgs { tzdata = true } svcs { - devfs = "sysinit" - dmesg = "sysinit" - hwdrivers = "sysinit" - mdev = "sysinit" - acpid = "boot" - bootmisc = "boot" - hostname = "boot" - hwclock = "boot" - modules = "boot" - swap = "boot" - sysctl = "boot" - syslog = "boot" - chronyd = "default" - networking = "default" - sshd = "default" - tiny-ec2-bootstrap = "default" - killprocs = "shutdown" - mount-ro = "shutdown" - savecache = "shutdown" + sysinit { + devfs = true + dmesg = true + hwdrivers = true + mdev = true + } + boot { + acpid = true + bootmisc = true + hostname = true + hwclock = true + modules = true + swap = true + sysctl = true + syslog = true + } + default { + chronyd = true + networking = true + sshd = true + tiny-ec2-bootstrap = true + } + shutdown { + killprocs = true + mount-ro = true + savecache = true + } } kernel_modules { sd-mod = true diff --git a/scripts/resolve-profile.py.in b/scripts/resolve-profile.py.in index 884cd2d..2905423 100644 --- a/scripts/resolve-profile.py.in +++ b/scripts/resolve-profile.py.in @@ -1,156 +1,133 @@ @PYTHON@ # vim: set ts=4 et: -import json import os -import shutil import sys -import boto3 -from botocore.exceptions import ClientError +import json +import shutil +import argparse from datetime import datetime, timedelta + from pyhocon import ConfigFactory -if len(sys.argv) != 2: - sys.exit("Usage: " + os.path.basename(__file__) + " ") - -PROFILE = sys.argv[1] - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -# path to the profile config file -PROFILE_CONF = os.path.join(SCRIPT_DIR, '..', 'profiles', PROFILE + '.conf') - -# load the profile's build configuration -BUILDS = ConfigFactory.parse_file(PROFILE_CONF)['BUILDS'] - -# where we store the profile's builds' config/output -PROFILE_DIR = os.path.join(SCRIPT_DIR, 'profile', PROFILE) -if not os.path.exists(PROFILE_DIR): - os.makedirs(PROFILE_DIR) - -# fold these build config keys' dict to scalar -FOLD_DICTS = { - 'ami_access': ',{0}', - 'ami_regions': ',{0}', - 'repos': "\n@{1} {0}", - 'pkgs': ' {0}@{1}', - 'kernel_modules': ',{0}', - 'kernel_options': ' {0}' -} - -NOW = datetime.utcnow() -ONE_DAY = timedelta(days=1) - - -# func to fold dict down to scalar -def fold(fdict, ffmt): - folded = '' - for fkey, fval in fdict.items(): - fkey = fkey.strip('"') # complex keys may be in quotes - if fval is True: - folded += ffmt[0] + fkey - elif fval not in [None, False]: - folded += ffmt.format(fkey, fval) - return folded[1:] - - -# list of AWS regions, and whether they're enabled -all_regions = {} -AWS = boto3.session.Session() -sys.stderr.write("\n>>> Determining region availability...") -sys.stderr.flush() -for region in AWS.get_available_regions('ec2'): - ec2 = AWS.client('ec2', region_name=region) - try: - ec2.describe_regions() - except ClientError as e: - if e.response['Error']['Code'] == 'AuthFailure': - sys.stderr.write('-') - sys.stderr.flush() - all_regions[region] = False - continue - elif e.response['Error']['Code'] == 'UnauthorizedOperation': - # have access to the region, but not to ec2:DescribeRegions - pass - else: - raise - sys.stderr.write('+') - sys.stderr.flush() - all_regions[region] = True -sys.stderr.write("\n") - -for region, available in all_regions.items(): - if available is False: - sys.stderr.write(f"*** WARNING: skipping disabled region {region}\n") - -print() - -# parse/resolve HOCON profile's builds' config -for build, cfg in BUILDS.items(): - print(f">>> Resolving configuration for '{build}'") - build_dir = os.path.join(PROFILE_DIR, build) - - # make a fresh profile build directory - if os.path.exists(build_dir): - shutil.rmtree(build_dir) - os.makedirs(build_dir) - - # populate profile build vars - cfg['profile'] = PROFILE - cfg['profile_build'] = build - - # mostly edge-related temporal substitutions - if cfg['end_of_life'] == '@TOMORROW@': - cfg['end_of_life'] = (NOW + ONE_DAY).isoformat(timespec='seconds') - elif cfg['end_of_life'] is not None: - # to explicitly UTC-ify end_of_life - cfg['end_of_life'] = datetime.fromisoformat( - cfg['end_of_life'] + '+00:00').isoformat(timespec='seconds') - if cfg['revision'] == '@NOW@': - cfg['revision'] = NOW.strftime('%Y%m%d%H%M%S') - - # 'ALL' region expansion (or retraction) - if 'ALL' in cfg['ami_regions']: - all_val = cfg['ami_regions']['ALL'] - if all_val not in [None, False]: - cfg['ami_regions'] = all_regions - else: - cfg['ami_regions'] = {} - else: - # warn/remove disabled regions - for region, enabled in all_regions.items(): - if enabled is not False or region not in cfg['ami_regions']: - continue - if cfg['ami_regions'][region] not in [None, False]: - cfg['ami_regions'][region] = False - - # fold dict vars to scalars - for foldkey, foldfmt in FOLD_DICTS.items(): - cfg[foldkey] = fold(cfg[foldkey], foldfmt) - - # fold 'svcs' dict to scalar - lvls = {} - for svc, lvl in cfg['svcs'].items(): - if lvl is True: - # service in default runlevel - lvls['default'].append(svc) - elif lvl not in [None, False]: - # service in specified runlevel (skip svc when false/null) - if lvl not in lvls.keys(): - lvls[lvl] = [] - lvls[lvl].append(svc) - cfg['svcs'] = ' '.join( - str(lvl) + '=' + ','.join( - str(svc) for svc in svcs - ) for lvl, svcs in lvls.items() - ) - - # resolve ami_name and ami_desc - cfg['ami_name'] = cfg['ami_name'].format(var=cfg) - cfg['ami_desc'] = cfg['ami_desc'].format(var=cfg) - - # write build vars file - with open(os.path.join(build_dir, 'vars.json'), 'w') as out: - json.dump(cfg, out, indent=4, separators=(',', ': ')) - -print() + +# Just group together our transforms +class Transforms: + + NOW = datetime.utcnow() + TOMORROW = NOW + timedelta(days=1) + + unquote = lambda x: x.strip('"') + + @staticmethod + def force_iso_date(input): + return datetime.fromisoformat(input).isoformat(timespec="seconds") + + @classmethod + def resolve_tomorrow(cls, input): + return cls.TOMORROW.isoformat(timespec="seconds") + + @classmethod + def resolve_now(cls, input): + return cls.NOW.strftime("%Y%m%d%H%M%S") + + @classmethod + def fold_comma(cls, input): + return ",".join([cls.unquote(k) for k in input.keys()]) + + @classmethod + def fold_space(cls, input): + return " ".join([cls.unquote(k) for k in input.keys()]) + + @classmethod + def fold_repos(cls, input): + return "\n".join( + f"@{v} {cls.unquote(k)}" if isinstance(v, str) else cls.unquote(k) + for k, v in input.items()) + + @staticmethod + def fold_packages(input): + return " ".join( + f"{k}@{v}" if isinstance(v, str) else k + for k, v in input.items()) + + @staticmethod + def fold_services(input): + return " ".join( + "{}={}".format(k, ",".join(v.keys())) + for k, v in input.items()) + + +class ConfigBuilder: + + _CFG_TRANSFORMS = { + "ami_access" : Transforms.fold_comma, + "ami_regions" : Transforms.fold_comma, + "kernel_modules" : Transforms.fold_comma, + "kernel_options" : Transforms.fold_space, + "repos" : Transforms.fold_repos, + "pkgs" : Transforms.fold_packages, + "svcs" : Transforms.fold_services, + "revision" : Transforms.resolve_now, + "end_of_life" : lambda x: \ + Transforms.force_iso_date(Transforms.resolve_tomorrow(x)), + } + + def __init__(self, config_path, out_dir): + self.config_path = config_path + self.out_dir = out_dir + + def build(self, profile): + build_config = ConfigFactory.parse_file(self.config_path) + + for build, cfg in build_config["BUILDS"].items(): + build_dir = os.path.join(self.out_dir, build) + + # Always start fresh + shutil.rmtree(build_dir, ignore_errors=True) + os.makedirs(build_dir) + + cfg["profile"] = profile + cfg["profile_build"] = build + + # Order of operations is important here + for k, v in cfg.items(): + transform = self._CFG_TRANSFORMS.get(k) + if transform: + cfg[k] = transform(v) + + if isinstance(v, str) and "{var." in v: + cfg[k] = v.format(var=cfg) + + with open(os.path.join(build_dir, "vars.json"), "w") as out: + json.dump(cfg, out, indent=4, separators=(",", ": ")) + + +def find_repo_root(): + path = os.getcwd() + + while ".git" not in set(os.listdir(path)) and path != "/": + path = os.path.dirname(path) + + if path == "/": + raise Exception("No repo found, stopping at /") + + return path + + +def main(args): + parser = argparse.ArgumentParser(description="Build Packer JSON variable " + "files from HOCON build profiles") + parser.add_argument("profile", help="name of profile to build") + args = parser.parse_args() + + root = find_repo_root() + + ConfigBuilder( + os.path.join(root, "profiles", f"{args.profile}.conf"), + os.path.join(root, "build", "profile", args.profile) + ).build(args.profile) + + +if __name__ == "__main__": + main(sys.argv) -- cgit v1.2.3