Refactor resolve-profile script

This is paving the way for identity broker improvements for opt-in
regions. The output is functionally identical between the two scripts
modulo the svcs change. Hopefully this makes the transformation process
a little more clear.

2 files changed, 152 insertions, 167 deletions

diff --git a/profiles/base/1 b/profiles/base/1
index fdda3f0..18decd1 100644
--- a/profiles/base/1
+++ b/profiles/base/1
@@ -56,25 +56,33 @@ pkgs {
   tzdata              = true
 }
 svcs {
-  devfs               = "sysinit"
-  dmesg               = "sysinit"
-  hwdrivers           = "sysinit"
-  mdev                = "sysinit"
-  acpid               = "boot"
-  bootmisc            = "boot"
-  hostname            = "boot"
-  hwclock             = "boot"
-  modules             = "boot"
-  swap                = "boot"
-  sysctl              = "boot"
-  syslog              = "boot"
-  chronyd             = "default"
-  networking          = "default"
-  sshd                = "default"
-  tiny-ec2-bootstrap  = "default"
-  killprocs           = "shutdown"
-  mount-ro            = "shutdown"
-  savecache           = "shutdown"
+  sysinit {
+    devfs               = true
+    dmesg               = true
+    hwdrivers           = true
+    mdev                = true
+  }
+  boot {
+    acpid               = true
+    bootmisc            = true
+    hostname            = true
+    hwclock             = true
+    modules             = true
+    swap                = true
+    sysctl              = true
+    syslog              = true
+  }
+  default {
+    chronyd             = true
+    networking          = true
+    sshd                = true
+    tiny-ec2-bootstrap  = true
+  }
+  shutdown {
+    killprocs           = true
+    mount-ro            = true
+    savecache           = true
+  }
 }
 kernel_modules {
   sd-mod      = true
diff --git a/scripts/resolve-profile.py.in b/scripts/resolve-profile.py.in
index 884cd2d..2905423 100644
--- a/scripts/resolve-profile.py.in
+++ b/scripts/resolve-profile.py.in
@@ -1,156 +1,133 @@
 @PYTHON@
 # vim: set ts=4 et:
 
-import json
 import os
-import shutil
 import sys
-import boto3
-from botocore.exceptions import ClientError
+import json
+import shutil
+import argparse
 from datetime import datetime, timedelta
+
 from pyhocon import ConfigFactory
 
-if len(sys.argv) != 2:
-    sys.exit("Usage: " + os.path.basename(__file__) + " <profile>")
-
-PROFILE = sys.argv[1]
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-
-# path to the profile config file
-PROFILE_CONF = os.path.join(SCRIPT_DIR, '..', 'profiles', PROFILE + '.conf')
-
-# load the profile's build configuration
-BUILDS = ConfigFactory.parse_file(PROFILE_CONF)['BUILDS']
-
-# where we store the profile's builds' config/output
-PROFILE_DIR = os.path.join(SCRIPT_DIR, 'profile', PROFILE)
-if not os.path.exists(PROFILE_DIR):
-    os.makedirs(PROFILE_DIR)
-
-# fold these build config keys' dict to scalar
-FOLD_DICTS = {
-    'ami_access': ',{0}',
-    'ami_regions': ',{0}',
-    'repos': "\n@{1} {0}",
-    'pkgs': ' {0}@{1}',
-    'kernel_modules': ',{0}',
-    'kernel_options': ' {0}'
-}
-
-NOW = datetime.utcnow()
-ONE_DAY = timedelta(days=1)
-
-
-# func to fold dict down to scalar
-def fold(fdict, ffmt):
-    folded = ''
-    for fkey, fval in fdict.items():
-        fkey = fkey.strip('"')    # complex keys may be in quotes
-        if fval is True:
-            folded += ffmt[0] + fkey
-        elif fval not in [None, False]:
-            folded += ffmt.format(fkey, fval)
-    return folded[1:]
-
-
-# list of AWS regions, and whether they're enabled
-all_regions = {}
-AWS = boto3.session.Session()
-sys.stderr.write("\n>>> Determining region availability...")
-sys.stderr.flush()
-for region in AWS.get_available_regions('ec2'):
-    ec2 = AWS.client('ec2', region_name=region)
-    try:
-        ec2.describe_regions()
-    except ClientError as e:
-        if e.response['Error']['Code'] == 'AuthFailure':
-            sys.stderr.write('-')
-            sys.stderr.flush()
-            all_regions[region] = False
-            continue
-        elif e.response['Error']['Code'] == 'UnauthorizedOperation':
-            # have access to the region, but not to ec2:DescribeRegions
-            pass
-        else:
-            raise
-    sys.stderr.write('+')
-    sys.stderr.flush()
-    all_regions[region] = True
-sys.stderr.write("\n")
-
-for region, available in all_regions.items():
-    if available is False:
-        sys.stderr.write(f"*** WARNING: skipping disabled region {region}\n")
-
-print()
-
-# parse/resolve HOCON profile's builds' config
-for build, cfg in BUILDS.items():
-    print(f">>> Resolving configuration for '{build}'")
-    build_dir = os.path.join(PROFILE_DIR, build)
-
-    # make a fresh profile build directory
-    if os.path.exists(build_dir):
-        shutil.rmtree(build_dir)
-    os.makedirs(build_dir)
-
-    # populate profile build vars
-    cfg['profile'] = PROFILE
-    cfg['profile_build'] = build
-
-    # mostly edge-related temporal substitutions
-    if cfg['end_of_life'] == '@TOMORROW@':
-        cfg['end_of_life'] = (NOW + ONE_DAY).isoformat(timespec='seconds')
-    elif cfg['end_of_life'] is not None:
-        # to explicitly UTC-ify end_of_life
-        cfg['end_of_life'] = datetime.fromisoformat(
-            cfg['end_of_life'] + '+00:00').isoformat(timespec='seconds')
-    if cfg['revision'] == '@NOW@':
-        cfg['revision'] = NOW.strftime('%Y%m%d%H%M%S')
-
-    # 'ALL' region expansion (or retraction)
-    if 'ALL' in cfg['ami_regions']:
-        all_val = cfg['ami_regions']['ALL']
-        if all_val not in [None, False]:
-            cfg['ami_regions'] = all_regions
-        else:
-            cfg['ami_regions'] = {}
-    else:
-        # warn/remove disabled regions
-        for region, enabled in all_regions.items():
-            if enabled is not False or region not in cfg['ami_regions']:
-                continue
-            if cfg['ami_regions'][region] not in [None, False]:
-                cfg['ami_regions'][region] = False
-
-    # fold dict vars to scalars
-    for foldkey, foldfmt in FOLD_DICTS.items():
-        cfg[foldkey] = fold(cfg[foldkey], foldfmt)
-
-    # fold 'svcs' dict to scalar
-    lvls = {}
-    for svc, lvl in cfg['svcs'].items():
-        if lvl is True:
-            # service in default runlevel
-            lvls['default'].append(svc)
-        elif lvl not in [None, False]:
-            # service in specified runlevel (skip svc when false/null)
-            if lvl not in lvls.keys():
-                lvls[lvl] = []
-            lvls[lvl].append(svc)
-    cfg['svcs'] = ' '.join(
-        str(lvl) + '=' + ','.join(
-            str(svc) for svc in svcs
-        ) for lvl, svcs in lvls.items()
-    )
-
-    # resolve ami_name and ami_desc
-    cfg['ami_name'] = cfg['ami_name'].format(var=cfg)
-    cfg['ami_desc'] = cfg['ami_desc'].format(var=cfg)
-
-    # write build vars file
-    with open(os.path.join(build_dir, 'vars.json'), 'w') as out:
-        json.dump(cfg, out, indent=4, separators=(',', ': '))
-
-print()
+
+# Just group together our transforms
+class Transforms:
+
+    NOW = datetime.utcnow()
+    TOMORROW = NOW + timedelta(days=1)
+
+    unquote = lambda x: x.strip('"')
+
+    @staticmethod
+    def force_iso_date(input):
+        return datetime.fromisoformat(input).isoformat(timespec="seconds")
+
+    @classmethod
+    def resolve_tomorrow(cls, input):
+        return cls.TOMORROW.isoformat(timespec="seconds")
+
+    @classmethod
+    def resolve_now(cls, input):
+        return cls.NOW.strftime("%Y%m%d%H%M%S")
+
+    @classmethod
+    def fold_comma(cls, input):
+        return ",".join([cls.unquote(k) for k in input.keys()])
+
+    @classmethod
+    def fold_space(cls, input):
+        return " ".join([cls.unquote(k) for k in input.keys()])
+
+    @classmethod
+    def fold_repos(cls, input):
+        return "\n".join(
+            f"@{v} {cls.unquote(k)}" if isinstance(v, str) else cls.unquote(k)
+            for k, v in input.items())
+
+    @staticmethod
+    def fold_packages(input):
+        return " ".join(
+            f"{k}@{v}" if isinstance(v, str) else k
+            for k, v in input.items())
+
+    @staticmethod
+    def fold_services(input):
+        return " ".join(
+            "{}={}".format(k, ",".join(v.keys()))
+            for k, v in input.items())
+
+
+class ConfigBuilder:
+
+    _CFG_TRANSFORMS = {
+        "ami_access"     : Transforms.fold_comma,
+        "ami_regions"    : Transforms.fold_comma,
+        "kernel_modules" : Transforms.fold_comma,
+        "kernel_options" : Transforms.fold_space,
+        "repos"          : Transforms.fold_repos,
+        "pkgs"           : Transforms.fold_packages,
+        "svcs"           : Transforms.fold_services,
+        "revision"       : Transforms.resolve_now,
+        "end_of_life"    : lambda x: \
+                Transforms.force_iso_date(Transforms.resolve_tomorrow(x)),
+    }
+
+    def __init__(self, config_path, out_dir):
+        self.config_path = config_path
+        self.out_dir = out_dir
+
+    def build(self, profile):
+        build_config = ConfigFactory.parse_file(self.config_path)
+
+        for build, cfg in build_config["BUILDS"].items():
+            build_dir = os.path.join(self.out_dir, build)
+
+            # Always start fresh
+            shutil.rmtree(build_dir, ignore_errors=True)
+            os.makedirs(build_dir)
+
+            cfg["profile"] = profile
+            cfg["profile_build"] = build
+
+            # Order of operations is important here
+            for k, v in cfg.items():
+                transform = self._CFG_TRANSFORMS.get(k)
+                if transform:
+                    cfg[k] = transform(v)
+
+                if isinstance(v, str) and "{var." in v:
+                    cfg[k] = v.format(var=cfg)
+
+            with open(os.path.join(build_dir, "vars.json"), "w") as out:
+                json.dump(cfg, out, indent=4, separators=(",", ": "))
+
+
+def find_repo_root():
+    path = os.getcwd()
+
+    while ".git" not in set(os.listdir(path)) and path != "/":
+        path = os.path.dirname(path)
+
+    if path == "/":
+        raise Exception("No repo found, stopping at /")
+
+    return path
+
+
+def main(args):
+    parser = argparse.ArgumentParser(description="Build Packer JSON variable "
+        "files from HOCON build profiles")
+    parser.add_argument("profile", help="name of profile to build")
+    args = parser.parse_args()
+
+    root = find_repo_root()
+
+    ConfigBuilder(
+        os.path.join(root, "profiles", f"{args.profile}.conf"),
+        os.path.join(root, "build", "profile", args.profile)
+    ).build(args.profile)
+
+
+if __name__ == "__main__":
+    main(sys.argv)