From cb95f7fd1eed7be5acbd386a95b0ba17823c6624 Mon Sep 17 00:00:00 2001 From: Jake Buchholz Date: Tue, 31 Jul 2018 17:55:39 -0700 Subject: Improve Configurability * move config variables from alpine-ami.yaml to variables.json-* + variables.json-default - ready-for-action original default config + variables.json-example - original defaults with comments * clean up tabs vs. spaces in make_ami.sh * make_ami.sh handles custom kernel flavor, extra repos, and extra packages * tweak README with regards to aws-ena-driver caveat --- README.md | 7 ++-- alpine-ami.yaml | 93 ++++++++++++++++++-------------------------------- make_ami.sh | 89 ++++++++++++++++++++++++----------------------- variables.json-default | 22 ++++++++++++ variables.json-example | 72 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 178 insertions(+), 105 deletions(-) create mode 100644 variables.json-default create mode 100644 variables.json-example diff --git a/README.md b/README.md index 5740f5c..0248607 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,10 @@ its development and thus there are some sharp edges. hardware so it seems unlikely that they will be supported going forward. Thus this project does not support them. -- The aws-ena-driver-vanilla package is still in edge/testing. When it is - available in a release, the edge/testing repository can be removed from - /etc/apk/repositories. +- The aws-ena-driver-vanilla package is still in edge/testing, and requires the + matching linux-vanilla package from edge/main. When ENA is available in an + alpine version release, edge/testing and edge/main should no longer be + necessary. - [cloud-init](https://cloudinit.readthedocs.io/en/latest/) is not currently supported on Alpine Linux. Instead this image uses diff --git a/alpine-ami.yaml b/alpine-ami.yaml index af97e49..bc39b89 100644 --- a/alpine-ami.yaml +++ b/alpine-ami.yaml @@ -1,84 +1,59 @@ variables: - security_group: "" - subnet: "" - public_ip: "false" - # Treat this similar to a ABUILD pkgrel variable and increment with every - # release. Packer will notice an exiting AMI at build start and fail unless - # it is rmoved. To prevent a period of time where no Alpine AMI exists, - # create a new variant. Old AMIs should be pruned at some point. - ami_release: "0" + # NOTE: Additional configuration is set via the `variables.json` file. + # To use default values, simply `cp variables.json-default variables.json`. + # See `variables.json-example` for full configuration variable descriptions. - # Overriding this requires validating that the installation script still - # works as expected. It probably does but stuff changes between major - # version. + # NOTE: Changing alpine_release requires modifying `make_ami.sh` -- don't + # override this in `variables.json`! alpine_release: "3.8" - # Don't override this without a good reason and if you do just make sure it - # gets passed all the way through to the make_ami script - volume_name: "/dev/xvdf" builders: - type: "amazon-ebssurrogate" - # Image is built inside a custom VPC so let Packer use the existing - # resources - security_group_id: "{{user `security_group`}}" - subnet_id: "{{user `subnet`}}" + ### Builder Instance Details - # Input Instance Setting - instance_type: "t2.nano" + vpc_id: "{{user `vpc`}}" + subnet_id: "{{user `subnet`}}" + security_group_id: "{{user `security_group`}}" + instance_type: "{{user `build_instance_type`}}" + associate_public_ip_address: "{{user `public_ip`}}" launch_block_device_mappings: - volume_type: "gp2" device_name: "{{user `volume_name`}}" - delete_on_termination: true - volume_size: 1 - associate_public_ip_address: "{{user `public_ip`}}" - - # Output AMI Settings - ena_support: true - ami_name: "Alpine-{{user `alpine_release`}}-r{{user `ami_release`}}-EC2" - ami_description: "Alpine Linux {{user `alpine_release`}}-r{{user `ami_release`}} Release with EC2 Optimizations" - ami_groups: - - "all" - ami_virtualization_type: "hvm" - ami_regions: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - ca-central-1 - - eu-central-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - ap-northeast-1 - - ap-northeast-2 -# - ap-northeast-3 - - ap-southeast-1 - - ap-southeast-2 - - ap-south-1 - - sa-east-1 - ami_root_device: - source_device_name: "{{user `volume_name`}}" - device_name: "/dev/xvda" - delete_on_termination: true - volume_size: 1 - volume_type: "gp2" - - # Use the most recent Amazon Linux AMI as our base + delete_on_termination: "true" + volume_size: "{{user `volume_size`}}" ssh_username: "ec2-user" source_ami_filter: + # use the latest Amazon Linux AMI filters: virtualization-type: "hvm" root-device-type: "ebs" architecture: "x86_64" name: "amzn-ami-hvm-*-x86_64-gp2" - owners: + owners: - "137112412989" - most_recent: true + most_recent: "true" + + ### Built AMI Details + + ami_name: "{{user `ami_name_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_name_suffix`}}" + ami_description: "{{user `ami_desc_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_desc_suffix`}}" + ami_virtualization_type: "hvm" + ami_root_device: + source_device_name: "{{user `volume_name`}}" + device_name: "/dev/xvda" + delete_on_termination: "true" + volume_size: "{{user `volume_size`}}" + volume_type: "gp2" + ena_support: "{{user `ena_enable`}}" + sriov_support: "{{user `sriov_enable`}}" + ami_groups: "{{user `ami_access`}}" + ami_regions: "{{user `deploy_regions`}}" + provisioners: - type: "shell" script: "make_ami.sh" - execute_command: "sudo sh -c '{{ .Vars }} {{ .Path }} {{user `volume_name`}}'" + execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }} {{user `volume_name`}} {{user `kernel_flavor`}} ''{{user `add_repos`}}'' ''{{user `add_pkgs`}}''"' diff --git a/make_ami.sh b/make_ami.sh index c85c160..1e5a0b3 100755 --- a/make_ami.sh +++ b/make_ami.sh @@ -1,9 +1,9 @@ #!/bin/sh -# vim:set ts=4: +# vim: set ts=4 noet: set -eu -: ${ALPINE_RELEASE:="3.8"} # not tested against edge +: ${ALPINE_RELEASE:="3.8"} # not tested against edge : ${APK_TOOLS_URI:="https://github.com/alpinelinux/apk-tools/releases/download/v2.10.0/apk-tools-2.10.0-x86_64-linux.tar.gz"} : ${APK_TOOLS_SHA256:="77f2d256fcd5d6fdafadf43bb6a9c85c3da7bb471ee842dcd729175235cb9fed"} : ${ALPINE_KEYS:="http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/alpine-keys-2.1-r1.apk"} @@ -19,9 +19,9 @@ einfo() { } rc_add() { - local target="$1"; shift # target directory + local target="$1"; shift # target directory local runlevel="$1"; shift # runlevel name - local services="$*" # names of services + local services="$*" # names of services local svc; for svc in $services; do mkdir -p "$target"/etc/runlevels/$runlevel @@ -31,9 +31,9 @@ rc_add() { } wgets() ( - local url="$1" # url to fetch - local sha256="$2" # expected SHA256 sum of output - local dest="$3" # output path and filename + local url="$1" # url to fetch + local sha256="$2" # expected SHA256 sum of output + local dest="$3" # output path and filename wget -T 10 -q -O "$dest" "$url" echo "$sha256 $dest" | sha256sum -c > /dev/null @@ -41,7 +41,7 @@ wgets() ( validate_block_device() { - local dev="$1" # target directory + local dev="$1" # target directory lsblk -P --fs "$dev" >/dev/null 2>&1 || \ die "'$dev' is not a valid block device" @@ -62,8 +62,8 @@ fetch_apk_tools() { } make_filesystem() { - local device="$1" # target device path - local target="$2" # mount target + local device="$1" # target device path + local target="$2" # mount target mkfs.ext4 "$device" e2label "$device" / @@ -71,15 +71,15 @@ make_filesystem() { } setup_repositories() { - local target="$1" # target directory + local target="$1" # target directory + local add_repos="$2" # extra repo lines, comma separated - # NOTE: we only need @testing for aws-ena-driver-vanilla, this can be removed if/when released mkdir -p "$target"/etc/apk/keys cat > "$target"/etc/apk/repositories <<-EOF http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_RELEASE/main http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_RELEASE/community - @testing http://dl-cdn.alpinelinux.org/alpine/edge/testing EOF + echo "$add_repos" | tr , "\012" >> "$target"/etc/apk/repositories } fetch_keys() { @@ -99,45 +99,44 @@ setup_chroot() { mount --bind /sys "$target"/sys # Don't want to ship this but it's needed for bootstrap. Will be removed in - # the cleanup stage. + # the cleanup stage. install -Dm644 /etc/resolv.conf "$target"/etc/resolv.conf } install_core_packages() { - local target="$1" + local target="$1" # target directory + local flavor="$2" # kernel flavor + local add_pkgs="$3" # extra packages, space separated # Most from: https://git.alpinelinux.org/cgit/alpine-iso/tree/alpine-virt.packages # - # acct - installed by some configurations, so added here - # aws-ena-driver-vanilla - required for ENA enabled instances (still in edge/testing) + # linux-$flavor - linux kernel flavor to install # e2fsprogs - required by init scripts to maintain ext4 volumes - # linux-vanilla - can't use virt because it's missing NVME support # mkinitfs - required to build custom initfs # sudo - to allow alpine user to become root, disallow root SSH logins # tiny-ec2-bootstrap - to bootstrap system from EC2 metadata chroot "$target" apk --no-cache add \ - acct \ + linux-"$flavor" \ alpine-mirrors \ - aws-ena-driver-vanilla@testing \ chrony \ e2fsprogs \ - linux-vanilla \ mkinitfs \ openssh \ sudo \ tiny-ec2-bootstrap \ - tzdata + tzdata \ + $add_pkgs chroot "$target" apk --no-cache add --no-scripts syslinux - # Disable starting getty for physical ttys because they're all inaccessible - # anyhow. With this configuration boot messages will still display in the - # EC2 console. - sed -Ei '/^tty\d/s/^/#/' /etc/inittab + # Disable starting getty for physical ttys because they're all inaccessible + # anyhow. With this configuration boot messages will still display in the + # EC2 console. + sed -Ei '/^tty\d/s/^/#/' /etc/inittab - # Make it a little more obvious who is logged in by adding username to the - # prompt - sed -i "s/^export PS1='/&\\\\u@/" /etc/profile + # Make it a little more obvious who is logged in by adding username to the + # prompt + sed -i "s/^export PS1='/&\\\\u@/" /etc/profile } create_initfs() { @@ -165,10 +164,10 @@ setup_extlinux() { # Enable ext4 because the root device is formatted ext4 # # Shorten timeout because EC2 has no way to interact with instance console - # - # ttyS0 is the target for EC2s "Get System Log" feature whereas tty0 is the - # target for EC2s "Get Instance Screenshot" feature. Enabling the serial - # port early in extlinux gives the most complete output in the system log. + # + # ttyS0 is the target for EC2s "Get System Log" feature whereas tty0 is the + # target for EC2s "Get Instance Screenshot" feature. Enabling the serial + # port early in extlinux gives the most complete output in the system log. sed -Ei -e "s|^[# ]*(root)=.*|\1=LABEL=/|" \ -e "s|^[# ]*(default_kernel_opts)=.*|\1=\"console=ttyS0 console=tty0\"|" \ -e "s|^[# ]*(serial_port)=.*|\1=ttyS0|" \ @@ -189,8 +188,8 @@ setup_fstab() { local target="$1" cat > "$target"/etc/fstab <<-EOF - # - LABEL=/ / ext4 defaults,noatime 1 1 + # + LABEL=/ / ext4 defaults,noatime 1 1 EOF } @@ -244,14 +243,14 @@ configure_ntp() { # in EC2. # # See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html - sed -i 's/^server .*/server 169.254.169.123/' "$target"/etc/chrony/chrony.conf + sed -i 's/^server .*/server 169.254.169.123/' "$target"/etc/chrony/chrony.conf } cleanup() { local target="$1" # Sweep cruft out of the image that doesn't need to ship or will be - # re-generated when the image boots + # re-generated when the image boots rm -f \ "$target"/var/cache/apk/* \ "$target"/etc/resolv.conf \ @@ -263,26 +262,30 @@ cleanup() { "$target"/proc \ "$target"/sys - umount "$target" + umount "$target" } main() { - [ "$#" -ne 1 ] && { echo "usage: $0 "; exit 1; } + [ "$#" -ne 4 ] && { echo "usage: $0 '[,]' '[ ]'"; exit 1; } device="$1" + flavor="$2" + add_repos="$3" + add_pkgs="$4" + target="/mnt/target" validate_block_device "$device" - [ -d "$target" ] || mkdir "$target" + [ -d "$target" ] || mkdir "$target" einfo "Fetching static APK tools" apk="$(fetch_apk_tools)" einfo "Creating root filesystem" - make_filesystem "$device" "$target" + make_filesystem "$device" "$target" - setup_repositories "$target" + setup_repositories "$target" "$add_repos" einfo "Fetching Alpine signing keys" fetch_keys "$target" @@ -293,7 +296,7 @@ main() { setup_chroot "$target" einfo "Installing core packages" - install_core_packages "$target" + install_core_packages "$target" "$flavor" "$add_pkgs" einfo "Configuring and enabling boot loader" create_initfs "$target" diff --git a/variables.json-default b/variables.json-default new file mode 100644 index 0000000..a1f3ce0 --- /dev/null +++ b/variables.json-default @@ -0,0 +1,22 @@ +{ + "ami_release": "1", + "ami_name_prefix": "Alpine-", + "ami_name_suffix": "-EC2", + "ami_desc_prefix": "Alpine Linux ", + "ami_desc_suffix": " Release with EC2 Optimizations", + "kernel_flavor": "vanilla@edge-main", + "add_repos": "@edge-main http://dl-cdn.alpinelinux.org/alpine/edge/main,@edge-testing http://dl-cdn.alpinelinux.org/alpine/edge/testing", + "add_pkgs": "acct aws-ena-driver-vanilla@edge-testing", + "ena_enable": "true", + "sriov_enable": "false", + "volume_size": "1", + "ami_access": "all", + "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1", + + "vpc": "", + "subnet": "", + "security_group": "", + "public_ip": "false", + "build_instance_type": "t2.nano", + "volume_name": "/dev/xvdf" +} diff --git a/variables.json-example b/variables.json-example new file mode 100644 index 0000000..4660c1d --- /dev/null +++ b/variables.json-example @@ -0,0 +1,72 @@ +# NOTE: This is file not valid JSON. +{ + ### Build Options ### + + # Treat similar to a ABUILD pkgrel variable and increment with every release. + "ami_release": "1", + + # AMI name prefix and suffix + "ami_name_prefix": "Alpine-", + "ami_name_suffix": "-EC2", + + # AMI description prefix and suffix + "ami_desc_prefix": "Alpine Linux ", + "ami_desc_suffix": " Release with EC2 Optimizations", + + # Kernel "flavor" to install. 'virt' is a slim choice, but doesn't currently + # include NVME support and there is no matching 'aws-ena-driver' package. + # 'vanilla' installs a lot of unneeded stuff (for an AMI), but does support + # NVME; however, there is no matching ENA driver in the main repo. In order + # to support NVME and ENA, we need to use 'vanilla@edge-main', which matches + # the 'aws-ena-driver@edge-testing' package. + "kernel_flavor": "vanilla@edge-main", + + # Comma separated list of lines to add to /etc/apk/repositories. We need + # edge/main and edge/testing for simultaneous NVME and ENA support. + "add_repos": "@edge-main http://dl-cdn.alpinelinux.org/alpine/edge/main,@edge-testing http://dl-cdn.alpinelinux.org/alpine/edge/testing", + + # Space separated list of additional packages to add to the AMI. + # acct - system accounting utilities (sa, etc.) + # aws-ena-driver-vanilla - Enhanced Network Adapter kernel module + "add_pkgs": "acct aws-ena-driver-vanilla@edge-testing", + + # Enable ENA/SRIOV support on the AMI. + "ena_enable": "true", + "sriov_enable": "false", + + # Size of the AMI image (in GiB). + "volume_size": "1", + + # Comma separated list of groups that should have access to the AMI. However, + # only two values are currently supported: 'all' for public, '' for private. + "ami_access": "all", + + # Comma separated list of regions to where the AMI should be copied. + # NOTE: ap-northeast-3 skipped, as it is available by subscription-only. + "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1", + + + ### Builder-Instance Options ### + + # VPC in which the builder instance is to be launched; you must also provide + # a subnet. + "vpc": "", + + # Subnet in which the builder instance is to be launched. + "subnet": "", + + # Security group to apply to the builder instance. + "security_group": "", + + # Assign a public IP to the builder instance. Set to 'true' for if you need + # to initiate the build from somewhere that wouldn't normally be able to + # access the builder instance's private network. + "public_ip": "false", + + # Instance type to use for building. + "build_instance_type": "t2.nano", + + # Don't override this without a good reason, and if you do just make sure it + # gets passed all the way through to the make_ami script. + "volume_name": "/dev/xvdf" +} -- cgit v1.2.3