Incorporate recent Alpine edge changes

* standardize on 'linux-virt' kernel
  + no longer need 'kernel_flavor' variable
  + always install 'aws-ena-driver' package
  + always enable ena_support
* switch to 'variables.yaml' for config
  + update build/convert script to stringify certain keys that may contain arrays
  + copy from 'variables.yaml-default' if it doesn't exist
* drop 'vpc' variable, using 'subnet' derives the proper VPC to use
* fix chrony.conf (all pool.ntp.org references are changed to 169.254.169.123)
* update README.md caveats

8 files changed, 97 insertions, 126 deletions

diff --git a/.gitignore b/.gitignore
index 00c9ff3..ca3d4df 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,5 @@
 /build/
 /.py3/
-/variables.json
+/variables.yaml
 /scrub-old-amis.py
 /gen-readme.py
diff --git a/Makefile b/Makefile
index 04ec2fb..d2505b2 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,9 @@
 .PHONY: ami
 ami: build/convert
+        [ -f variables.yaml ] || cp variables.yaml-default variables.yaml
+        build/convert variables.yaml > build/variables.json
         build/convert alpine-ami.yaml > build/alpine-ami.json
-        packer build -var-file=variables.json build/alpine-ami.json
+        packer build -var-file=build/variables.json build/alpine-ami.json
 
 build/convert:
         [ -d ".py3" ] || python3 -m venv .py3
@@ -14,7 +16,11 @@ build/convert:
         # which only supports JSON
         @echo "#!`pwd`/.py3/bin/python" > build/convert
         @echo "import yaml, json, sys" >> build/convert
-        @echo "json.dump(yaml.load(open(sys.argv[1])), sys.stdout, indent=4, separators=(',', ': '))" >> build/convert
+        @echo "y = yaml.load(open(sys.argv[1]))" >> build/convert
+        @echo "for k in ['ami_access','deploy_regions','add_repos','add_pkgs']:" >> build/convert
+        @echo "  if k in y and isinstance(y[k], list):" >> build/convert
+        @echo "    y[k] = ','.join(str(x) for x in y[k])" >> build/convert
+        @echo "json.dump(y, sys.stdout, indent=4, separators=(',', ': '))" >> build/convert
         @chmod +x build/convert
 
 %.py: %.py.in
diff --git a/README.md b/README.md
index 10a1322..a27c2f3 100644
--- a/README.md
+++ b/README.md
@@ -50,16 +50,6 @@ its development and thus there are some sharp edges.
   hardware so it seems unlikely that they will be supported going forward. Thus
   this project does not support them.
 
-- The linux-vanilla kernel all the linux-firmware packages it installs is much
-  larger than is necessary for an AMI designed to run on EC2.  Unfortunately,
-  the linux-virt kernel is currently missing NVMe support, which is required for
-  the newest generation of instance families.
-
-- The aws-ena-driver-vanilla package is still in edge/testing, and requires the
-  matching linux-vanilla package from edge/main.  When ENA is available in an
-  alpine version release (ideally with a 'virt' kernel flavor), edge/testing
-  and edge/main should no longer be necessary.
-
 - [cloud-init](https://cloudinit.readthedocs.io/en/latest/) is not currently
   supported on Alpine Linux. Instead this image uses
   [tiny-ec2-bootstrap](https://github.com/mcrute/tiny-ec2-bootstrap). Hostname
@@ -71,6 +61,13 @@ its development and thus there are some sharp edges.
   If full cloud-init support is important to you please file a bug against this
   project.
 
+- Because several key packages in Alpine 3.8 are missing or lacking features,
+  we currently need to install some packages from edge.  We expect that these
+  will be included in Alpine 3.9, or perhaps as a 3.8.x update.
+  - linux-virt @edge-main (includes necessary NVMe drivers)
+  - aws-ena-driver @edge-testing (installs 'virt' flavored subpackage)
+  - tiny-ec2-bootstrap @edge-main  (updated to v1.2.0)
+
 - CloudFormation support is still forthcoming. This requires patches and
   packaging for the upstream cfn tools that have not yet been accepted.
   Eventually full CloudFormation support will be available.
diff --git a/alpine-ami.yaml b/alpine-ami.yaml
index 09e0c4d..0ec2e21 100644
--- a/alpine-ami.yaml
+++ b/alpine-ami.yaml
@@ -1,8 +1,7 @@
 variables:
 
-  # NOTE:  Configuration is done with a `variables.json` file.
-  # To use default values, simply `cp variables.json-default variables.json`.
-  # See `variables.json-example` for full configuration variable descriptions.
+  # NOTE:  Configuration is done with a `variables.yaml` file.  If it doesn't
+  # exist, default configuration is copied from `variables.yaml-default`.
 
   # NOTE:  Changing alpine_release requires modifying `make_ami.sh` -- don't
   # override this in `variables.json`!
@@ -14,7 +13,6 @@ builders:
 
     ### Builder Instance Details
 
-    vpc_id: "{{user `vpc`}}"
     subnet_id: "{{user `subnet`}}"
     security_group_id: "{{user `security_group`}}"
     instance_type: "t3.nano"
@@ -36,7 +34,7 @@ builders:
         - "137112412989"
       most_recent: "true"
 
-    ### Built AMI Details
+    ### AMI Build Details
 
     ami_name: "{{user `ami_name_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_name_suffix`}}"
     ami_description: "{{user `ami_desc_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_desc_suffix`}}"
@@ -48,7 +46,7 @@ builders:
       volume_size: "{{user `volume_size`}}"
       volume_type: "gp2"
     encrypt_boot: "{{user `encrypt_ami`}}"
-    ena_support: "{{user `ena_enable`}}"
+    ena_support: "true"
     sriov_support: "true"
     ami_groups: "{{user `ami_access`}}"
     ami_regions: "{{user `deploy_regions`}}"
@@ -57,4 +55,4 @@ builders:
 provisioners:
   - type: "shell"
     script: "make_ami.sh"
-    execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }} {{user `kernel_flavor`}} ''{{user `add_repos`}}'' ''{{user `add_pkgs`}}''"'
+    execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }} ''{{user `add_repos`}}'' ''{{user `add_pkgs`}}''"'
diff --git a/make_ami.sh b/make_ami.sh
index 6218694..0d10058 100755
--- a/make_ami.sh
+++ b/make_ami.sh
@@ -108,8 +108,7 @@ setup_chroot() {
 
 install_core_packages() {
     local target="$1"    # target directory
-    local flavor="$2"    # kernel flavor
-    local add_pkgs="$3"  # extra packages, space separated
+    local add_pkgs="$2"  # extra packages, space separated
 
     # Most from: https://git.alpinelinux.org/cgit/alpine-iso/tree/alpine-virt.packages
     #
@@ -117,15 +116,15 @@ install_core_packages() {
     # tiny-ec2-bootstrap - to bootstrap system from EC2 metadata
     #
     chroot "$target" apk --no-cache add \
-        linux-"$flavor" \
+        linux-virt@edge-main \
+        aws-ena-driver@edge-testing \
         alpine-mirrors \
         chrony \
-        e2fsprogs \
         openssh \
         sudo \
-        tiny-ec2-bootstrap \
+        tiny-ec2-bootstrap@edge-main \
         tzdata \
-        $add_pkgs
+        $(echo "$add_pkgs" | tr , ' ')
 
     chroot "$target" apk --no-cache add --no-scripts syslinux
 
@@ -244,7 +243,9 @@ configure_ntp() {
     # in EC2.
     #
     # See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html
-    sed -i 's/^pool .*/server 169.254.169.123 iburst/' "$target"/etc/chrony/chrony.conf
+    sed -e 's/^pool /server /' \
+        -e 's/pool.ntp.org/169.254.169.123/g' \
+        -i "$target"/etc/chrony/chrony.conf
 }
 
 cleanup() {
@@ -267,11 +268,10 @@ cleanup() {
 }
 
 main() {
-    [ "$#" -ne 3 ] && { echo "usage: $0 <kernel-flavor> '<repo>[,<repo>]' '<pkg>[ <pkg>]'"; exit 1; }
+    [ "$#" -ne 2 ] && { echo "usage: $0 '<repo>[,<repo>]' '<pkg>[,<pkg>]'"; exit 1; }
 
-    local flavor="$1"
-    local add_repos="$2"
-    local add_pkgs="$3"
+    local add_repos="$1"
+    local add_pkgs="$2"
 
     local device="/dev/xvdf"
     local target="/mnt/target"
@@ -297,7 +297,7 @@ main() {
     setup_chroot "$target"
 
     einfo "Installing core packages"
-    install_core_packages "$target" "$flavor" "$add_pkgs"
+    install_core_packages "$target" "$add_pkgs"
 
     einfo "Configuring and enabling boot loader"
     create_initfs "$target"
diff --git a/variables.json-default b/variables.json-default
deleted file mode 100644
index d45378f..0000000
--- a/variables.json-default
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-  "ami_release": "1",
-  "ami_name_prefix": "Alpine-",
-  "ami_name_suffix": "-EC2",
-  "ami_desc_prefix": "Alpine Linux ",
-  "ami_desc_suffix": " Release with EC2 Optimizations",
-  "kernel_flavor": "vanilla@edge-main",
-  "add_repos": "",
-  "add_pkgs": "aws-ena-driver-vanilla@edge-testing",
-  "ena_enable": "true",
-  "volume_size": "1",
-  "encrypt_ami": "false",
-  "ami_access": "all",
-  "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1",
-
-  "vpc": "",
-  "subnet": "",
-  "security_group": "",
-  "public_ip": "false"
-}
diff --git a/variables.json-example b/variables.json-example
deleted file mode 100644
index 0e10c9a..0000000
--- a/variables.json-example
+++ /dev/null
@@ -1,74 +0,0 @@
-# *** NOTE: This is file not valid JSON! ***
-
-{
-  ### Build Options ###
-
-  # Treat similar to a ABUILD pkgrel variable and increment with every release.
-  "ami_release": "1",
-
-  # AMI name prefix and suffix
-  "ami_name_prefix": "Alpine-",
-  "ami_name_suffix": "-EC2",
-
-  # AMI description prefix and suffix
-  "ami_desc_prefix": "Alpine Linux ",
-  "ami_desc_suffix": " Release with EC2 Optimizations",
-
-  # Kernel "flavor" to install.
-  #
-  # 'virt' is the slim choice, but doesn't currently include NVMe support and
-  # there is no matching 'aws-ena-driver' package.  When these features are
-  # available, this kernel flavor will be the default (if not hardcoded).
-  #
-  # 'vanilla' installs a lot of unneeded stuff (for an AMI), but does support
-  # NVMe; however, there is no matching ENA driver in the main repo.  In order
-  # to support NVMe and ENA, we need to use 'vanilla@edge-main', which matches
-  # the 'aws-ena-driver@edge-testing' package.
-  #
-  "kernel_flavor": "vanilla@edge-main",
-
-  # Comma separated list of custom lines to add to /etc/apk/repositories.
-  # @edge-main, @edge-community, and @edge-testing repos have been predefined.
-  "add_repos": "",
-
-  # Space separated list of additional packages to add to the AMI.
-  # aws-ena-driver-vanilla - ENA driver (until we have a 'virt' flavor)
-  "add_pkgs": "aws-ena-driver-vanilla@edge-testing",
-
-  # Enable ENA support on the AMI.
-  # When ENA is available for the 'virt' kernel, this will always be on.
-  "ena_enable": "true",
-
-  # Size of the AMI image (in GiB).
-  "volume_size": "1",
-
-  # Encrypt the AMI?
-  "encrypt_ami": "false",
-
-  # Comma separated list of groups that should have access to the AMI.  However,
-  # only two values are currently supported: 'all' for public, '' for private.
-  "ami_access": "all",
-
-  # Comma separated list of regions to where the AMI should be copied.
-  # NOTE: ap-northeast-3 skipped, as it is available by subscription-only.
-  "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1",
-
-
-  ### Builder-Instance Options ###
-
-  # VPC in which the builder instance is to be launched; you must also provide
-  # a subnet.
-  "vpc": "",
-
-  # Subnet in which the builder instance is to be launched.
-  "subnet": "",
-
-  # Security group to apply to the builder instance.
-  "security_group": "",
-
-  # Assign a public IP to the builder instance.  Set to 'true' for if you need
-  # to initiate the build from somewhere that wouldn't normally be able to
-  # access the builder instance's private network.
-  "public_ip": "false"
-
-}
diff --git a/variables.yaml-default b/variables.yaml-default
new file mode 100644
index 0000000..3ec54f4
--- /dev/null
+++ b/variables.yaml-default
@@ -0,0 +1,64 @@
+### Builder-Instance Options ###
+
+# Subnet ID in which the builder instance is to be launched.  VPC will be
+# automatically determined.
+subnet:
+
+# Optional security group to apply to the builder instance.
+security_group:
+
+# Assign a public IP to the builder instance.  Set to "true" for if you need
+# to initiate the build from somewhere that wouldn't normally be able to access
+# the builder instance's private network and a private IP is not auto-assigned.
+public_ip: "false"
+
+
+### Build Options ###
+
+# Treat similar to a ABUILD pkgrel variable and increment with every release.
+ami_release: "1"
+
+# AMI name prefix and suffix
+ami_name_prefix: "Alpine-"
+ami_name_suffix: "-EC2"
+
+# AMI description prefix and suffix
+ami_desc_prefix: "Alpine Linux "
+ami_desc_suffix: " Release with EC2 Optimizations"
+
+# List of custom lines to add to /etc/apk/repositories.  Note that @edge-main,
+# @edge-community, and @edge-testing repos have been predefined.
+add_repos:
+
+# List of additional packages to add to the AMI.
+add_pkgs:
+
+# Size of the AMI image (in GiB).
+volume_size: "1"
+
+# Encrypt the AMI?
+encrypt_ami: "false"
+
+# List of groups that should have access to the AMI.  However, only two
+# values are currently supported: 'all' for public, '' or unset for private.
+ami_access:
+  - "all"
+
+# List of regions to where the AMI should be copied.
+deploy_regions:
+  - "us-east-1"
+  - "us-east-2"
+  - "us-west-1"
+  - "us-west-2"
+  - "ca-central-1"
+  - "eu-central-1"
+  - "eu-west-1"
+  - "eu-west-2"
+  - "eu-west-3"
+  - "ap-northeast-1"
+  - "ap-northeast-2"
+#  - "ap-northeast-3" # skipped, available by subscription only
+  - "ap-southeast-1"
+  - "ap-southeast-2"
+  - "ap-south-1"
+  - "sa-east-1"