aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJake Buchholz <tomalok@gmail.com>2019-05-27 22:27:55 -0700
committerMike Crute <mike@crute.us>2019-07-05 12:51:09 -0700
commit396bb8ab867d46217c943c0387979e862b9a05d5 (patch)
tree66ab70fc243115a55029b846e7937fb440f1eb5b
parent24144391d61723da5217539f7b22b5ea37b959f0 (diff)
downloadalpine-ec2-ami-396bb8ab867d46217c943c0387979e862b9a05d5.tar.bz2
alpine-ec2-ami-396bb8ab867d46217c943c0387979e862b9a05d5.tar.xz
alpine-ec2-ami-396bb8ab867d46217c943c0387979e862b9a05d5.zip
Build Profiles and 3.9.4
* Build Profiles (completion of PR #49) + auto-updates version profile when new release detected + updates releases/<profile>.yaml after successful builds * Prune AMIs (in AWS and in releases/<profile>.yaml + 'revision' - keep latest revision per release + 'release' - keep latest release per version + 'version' - remove end-of-life versions * releases/README.md updater script * README overhaul + Pre-built AMIs --> releases/README.md + profiles/README.md for profile configuration details + main README.md overhauled to go over how to build and manage custom AMIs
Diffstat
-rw-r--r--.gitignore17
-rw-r--r--LICENSE.txt2
-rw-r--r--Makefile81
-rw-r--r--README.md177
-rw-r--r--alpine-ami.yaml70
-rw-r--r--gen-readme.py.in17
-rwxr-xr-xmake_ami.sh368
-rw-r--r--packer.conf108
-rw-r--r--profiles/README.md157
-rw-r--r--profiles/alpine.conf47
l---------profiles/arch/aarch641
-rw-r--r--profiles/arch/aarch64-110
l---------profiles/arch/x86_641
-rw-r--r--profiles/arch/x86_64-19
-rw-r--r--profiles/base/187
l---------profiles/base/current1
-rw-r--r--profiles/test.conf31
-rw-r--r--profiles/version/3.914
l---------profiles/version/current1
-rw-r--r--profiles/version/edge18
-rw-r--r--release.yaml22
-rw-r--r--releases/README.md77
-rw-r--r--releases/alpine.yaml58
-rw-r--r--scripts/gen-release-readme.py.in115
-rwxr-xr-xscripts/make-amis61
-rwxr-xr-xscripts/nvme/nvme-ebs-links (renamed from nvme/nvme-ebs-links)15
-rw-r--r--scripts/nvme/nvme-ebs-mdev.conf (renamed from nvme/nvme-ebs-mdev.conf)0
-rw-r--r--scripts/prune-amis.py.in133
-rw-r--r--scripts/resolve-profile.py.in105
-rwxr-xr-xscripts/setup-ami344
-rw-r--r--scripts/update-release.py.in62
-rw-r--r--scrub-old-amis.py.in72
-rw-r--r--variables.yaml-default77
33 files changed, 1609 insertions, 749 deletions
diff --git a/.gitignore b/.gitignore
index efc91a3..89f22ab 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,15 @@
1**/*~ 1**/*~
2**/*.bak
2**/*.swp 3**/*.swp
3/build/ 4/build/
4/.py3/ 5/profiles/*
5/variables.yaml 6!/profiles/README.md
6/variables.yaml_* 7!/profiles/base/
7/scrub-old-amis.py 8!/profiles/arch/
8/gen-readme.py 9!/profiles/version/
10!/profiles/alpine.conf
11!/profiles/example.conf
12!/profiles/test.conf
13/releases/*
14!/releases/README.md
15!/releases/alpine.yaml
diff --git a/LICENSE.txt b/LICENSE.txt
index 736d3fe..bb205ac 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,4 +1,4 @@
1Copyright (c) 2017 Michael Crute 1Copyright (c) 2017-2019 Michael Crute, Jake Buchholz
2 2
3Permission is hereby granted, free of charge, to any person obtaining a copy of 3Permission is hereby granted, free of charge, to any person obtaining a copy of
4this software and associated documentation files (the "Software"), to deal in 4this software and associated documentation files (the "Software"), to deal in
diff --git a/Makefile b/Makefile
index a624e9e..626f6c8 100644
--- a/Makefile
+++ b/Makefile
@@ -1,44 +1,43 @@
1.PHONY: ami 1# vim: ts=8 noet:
2 2
3ami: convert 3ALL_SCRIPTS := $(wildcard scripts/*)
4 packer build -var-file=build/variables.json build/alpine-ami.json 4CORE_PROFILES := $(wildcard profiles/*/*)
5 5TARGET_PROFILES := $(wildcard profiles/*.conf)
6edge: convert 6PROFILE :=
7 @echo '{ "version": "edge", "release": "edge", "revision": "'-`date +%Y%m%d%H%M%S`'" }' > build/edge.json 7BUILD :=
8 packer build -var-file=build/variables.json -var-file=build/edge.json build/alpine-ami.json 8BUILDS := $(BUILD)
9 9LEVEL :=
10convert: build/convert 10
11 [ -f variables.yaml ] || cp variables.yaml-default variables.yaml 11# by default, use the 'packer' in the path
12 build/convert variables.yaml > build/variables.json 12PACKER := packer
13 build/convert alpine-ami.yaml > build/alpine-ami.json 13export PACKER
14 14
15build/convert: 15.PHONY: amis prune release-readme clean
16 [ -d ".py3" ] || python3 -m venv .py3 16
17 .py3/bin/pip install pyyaml boto3 17amis: build build/packer.json build/profile/$(PROFILE) build/update-release.py
18 18 build/make-amis $(PROFILE) $(BUILDS)
19 [ -d "build" ] || mkdir build 19
20 20prune: build build/prune-amis.py
21 # Make stupid simple little YAML/JSON converter so we can maintain our 21 build/prune-amis.py $(LEVEL) $(PROFILE) $(BUILD)
22 # packer configs in a sane format that allows comments but also use packer 22
23 # which only supports JSON 23release-readme: build build/gen-release-readme.py
24 @echo "#!`pwd`/.py3/bin/python" > build/convert 24 build/gen-release-readme.py $(PROFILE)
25 @echo "import yaml, json, sys" >> build/convert 25
26 @echo "y = yaml.full_load(open(sys.argv[1]))" >> build/convert 26build: $(SCRIPTS)
27 @echo "for k in ['ami_access','deploy_regions','add_repos','add_pkgs','add_svcs']:" >> build/convert 27 [ -d build/profile ] || mkdir -p build/profile
28 @echo " if k in y and isinstance(y[k], list):" >> build/convert 28 python3 -m venv build/.py3
29 @echo " y[k] = ','.join(str(x) for x in y[k])" >> build/convert 29 build/.py3/bin/pip install pyhocon pyyaml boto3
30 @echo " if k in y and isinstance(y[k], dict):" >> build/convert 30 (cd build; for i in $(ALL_SCRIPTS); do ln -sf ../$$i .; done)
31 @echo " y[k] = ':'.join(str(l) + '=' + ','.join(str(s) for s in ss) for l, ss in y[k].items())" >> build/convert 31
32 @echo "json.dump(y, sys.stdout, indent=4, separators=(',', ': '))" >> build/convert 32build/packer.json: build packer.conf
33 @chmod +x build/convert 33 build/.py3/bin/pyhocon -i packer.conf -f json > build/packer.json
34 34
35%.py: %.py.in 35build/profile/$(PROFILE): build build/resolve-profile.py $(CORE_PROFILES) $(TARGET_PROFILES)
36 sed "s|@PYTHON@|#!`pwd`/.py3/bin/python|" $< > $@ 36 build/resolve-profile.py $(PROFILE)
37
38%.py: %.py.in build
39 sed "s|@PYTHON@|#!`pwd`/build/.py3/bin/python|" $< > $@
37 chmod +x $@ 40 chmod +x $@
38 41
39.PHONY: clean
40clean: 42clean:
41 rm -rf build .py3 scrub-old-amis.py gen-readme.py 43 rm -rf build
42
43distclean: clean
44 rm -f variables.yaml
diff --git a/README.md b/README.md
index 0e3ae25..a08da13 100644
--- a/README.md
+++ b/README.md
@@ -1,72 +1,111 @@
1# Alpine Linux EC2 AMI Build 1# Alpine Linux EC2 AMI Builder
2 2
3**NOTE: This is not an official Amazon or AWS provided image. This is 3**NOTE: This is not an official AWS or Alpine project. This is community built
4community built and supported.** 4and supported.**
5 5
6This repository contains a packer file and a script to create an EC2 AMI 6## Pre-Built AMIs
7containing Alpine Linux. The AMI is designed to work with most EC2 features 7
8such as Elastic Network Adapters and NVME EBS volumes by default. If anything 8***To get started with one of our pre-built minimalist AMIs, please refer to the
9is missing please report a bug. 9[README](releases/README.md) in the [releases](releases) subdirectory.***
10 10
11This image can be launched on any modern x86_64 instance type, including T3, 11## Custom AMIs
12M5, C5, I3, R5, P3, X1, X1e, D2, Z1d. Other instances may also work but have 12
13not been tested. If you find an issue with instance support for any current 13Using the scripts and configuration in this project, you can build your own
14generation instance please file a bug against this project. 14custom Alpine Linux AMIs. If you experience any problems building custom AMIs,
15 15please open an [issue](https://github.com/mcrute/alpine-ec2-ami/issues) and
16To get started use one of the AMIs below. The default user is `alpine` and 16include as much detailed information as possible.
17will be configured to use whatever SSH keys you chose when you launched the 17
18image. If user data is specified it must be a shell script that begins with 18### Build Requirements
19`#!`. If a script is provided it will be executed as root after the network is 19
20configured. 20* [Packer](https://packer.io) >= 1.4.1
21 21* [Python 3.x](https://python.org) (3.7 is known to work)
22**NOTE:** *We are working to automate AMI builds and updates to this file and 22* `make` (GNU Make is known to work)
23[release.yaml](https://github.com/mcrute/alpine-ec2-ami/blob/master/release.yaml) 23* an AWS account with an existing subnet in an AWS Virtual Private Cloud
24in the not-too-distant future.* 24
25 25### Profile Configuration
26| Alpine Release | Region Code | AMI ID | 26
27| :------------: | ----------- | ------ | 27Target profile config files reside in the [profiles](profiles) subdirectory,
28| 3.9.3 | ap-northeast-1 | [ami-001e74131496d0212](https://ap-northeast-1.console.aws.amazon.com/ec2/home#launchAmi=ami-001e74131496d0212) | 28where you will also find the [config](profiles/alpine.conf) we use for our
29| 3.9.3 | ap-northeast-2 | [ami-09a26b03424d75667](https://ap-northeast-2.console.aws.amazon.com/ec2/home#launchAmi=ami-09a26b03424d75667) | 29pre-built AMIs. Refer to the [README](profiles/README.md) in that subdirectory
30| 3.9.3 | ap-south-1 | [ami-03534f64f8b87aafc](https://ap-south-1.console.aws.amazon.com/ec2/home#launchAmi=ami-03534f64f8b87aafc) | 30for more details and information about how AMI profile configs work.
31| 3.9.3 | ap-southeast-1 | [ami-0d5f2950efcd55b0e](https://ap-southeast-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0d5f2950efcd55b0e) | 31
32| 3.9.3 | ap-southeast-2 | [ami-0660edcba4ba7c8a0](https://ap-southeast-2.console.aws.amazon.com/ec2/home#launchAmi=ami-0660edcba4ba7c8a0) | 32### AWS Credentials
33| 3.9.3 | ca-central-1 | [ami-0bf4ea1f0f86283bb](https://ca-central-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0bf4ea1f0f86283bb) | 33
34| 3.9.3 | eu-central-1 | [ami-060d9bbde8d5047e8](https://eu-central-1.console.aws.amazon.com/ec2/home#launchAmi=ami-060d9bbde8d5047e8) | 34These scripts use the `boto3` library to interact with AWS, enabling you to
35| 3.9.3 | eu-north-1 | [ami-0a5284750fcf11d18](https://eu-north-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0a5284750fcf11d18) | 35provide your AWS account credentials in a number of different ways. see the
36| 3.9.3 | eu-west-1 | [ami-0af60b964eb2f09d3](https://eu-west-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0af60b964eb2f09d3) | 36offical `boto3` documentation's section on
37| 3.9.3 | eu-west-2 | [ami-097405edd3790cf8b](https://eu-west-2.console.aws.amazon.com/ec2/home#launchAmi=ami-097405edd3790cf8b) | 37[configuring credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#configuring-credentials)
38| 3.9.3 | eu-west-3 | [ami-0078916a37514bb9a](https://eu-west-3.console.aws.amazon.com/ec2/home#launchAmi=ami-0078916a37514bb9a) | 38for more details. *Please note that these scripts do not implement the first
39| 3.9.3 | sa-east-1 | [ami-09e0025e60328ea6d](https://sa-east-1.console.aws.amazon.com/ec2/home#launchAmi=ami-09e0025e60328ea6d) | 39two methods on the list.*
40| 3.9.3 | us-east-1 | [ami-05c8c48601c2303af](https://us-east-1.console.aws.amazon.com/ec2/home#launchAmi=ami-05c8c48601c2303af) | 40
41| 3.9.3 | us-east-2 | [ami-064d64386a89de1e6](https://us-east-2.console.aws.amazon.com/ec2/home#launchAmi=ami-064d64386a89de1e6) | 41### Building AMIs
42| 3.9.3 | us-west-1 | [ami-04a4711d62db12ba0](https://us-west-1.console.aws.amazon.com/ec2/home#launchAmi=ami-04a4711d62db12ba0) | 42
43| 3.9.3 | us-west-2 | [ami-0ff56870cf29d4f02](https://us-west-2.console.aws.amazon.com/ec2/home#launchAmi=ami-0ff56870cf29d4f02) | 43To build all build targets in a target profile, simply...
44```
45make PROFILE=<profile>
46```
47
48You can also build specfic build targets within a profile:
49```
50make PROFILE=<profile> BUILDS="<build1> <build2>"
51```
52
53If the `packer` binary is not in your `PATH`, or you would like to specify a
54different one, use...
55```
56make PACKER=<packer-path> PROFILE=<profile>
57```
58
59Before each build, new Alpine Linux *releases* are detected and the version's
60core profile is updated.
61
62If there's already an AMI with the same name as the profile build's, that build
63will be skipped and the process moves on to build the other profile's build
64targets (if any).
65
66After each successful build, `releases/<profile>.yaml` is updated with the
67build's details, including (most importantly) the ids of the AMI artifacts that
68were built.
69
70Additional information about using your custom AMIs can be found in the
71[README](releases/README.md) in the [releases](releases) subdirectory.
72
73### Pruning AMIs
74
75Every now and then, you may want to clean up old AMIs from your EC2 account and
76your profile's `releases/<profile>.yaml`. There are three different levels of
77pruning:
78* `revision` - keep only the latest revision for each release
79* `release` - keep only the latest release for each version
80* `version` - remove any end-of-life versions
81
82To prune a profile (or optionally one build target of a profile...
83```
84make prune LEVEL=<level> PROFILE=<profile> [BUILD=<build>]
85```
86
87Any AMIs in the account which are "unknown" (to the profile/build target, at
88least) will be called out as such, but will not be pruned.
89
90### Updating the Release README
91
92This make target updates the [releases README](releases/README.md), primarily
93for updating the list of our pre-built AMIs. This may-or-may-not be useful for
94other target profiles.
95```
96make release-readme PROFILE=<profile>
97```
98
99### Cleaning up the Build Environment
100
101`make clean` will remove the temporary `build` subdirectory, which contains the
102resolved profile and Packer configs, the Python virtual environment, and other
103temporary build-related artifacts.
44 104
45## Caveats 105## Caveats
46 106
47This image is being used in production but it's still somewhat early stage in 107* New Alpine Linux *versions* are currently not auto-detected and added as a
48its development and thus there are some sharp edges. 108 core version profile; this process is, at the moment, still a manual task.
49 109
50- As of 3.9.0-1, this AMI starts `haveged` at the boot runlevel, to provide 110* Although it's possible to build "aarch64" (arm64) AMIs, they don't quite work
51 additional initial entropy as discussed in issue #39. In the long term, we 111 yet.
52 hope to find an alternative solution.
53
54- Only EBS-backed HVM instances are supported. While paravirtualized instances
55 are still available from AWS they are not supported on any of the newer
56 hardware so it seems unlikely that they will be supported going forward.
57 Thus this project does not support them.
58
59- [cloud-init](https://cloudinit.readthedocs.io/en/latest/) is not currently
60 supported on Alpine Linux. Instead this image uses
61 [tiny-ec2-bootstrap](https://github.com/mcrute/tiny-ec2-bootstrap). Hostname
62 setting will work, as will setting the ssh keys for the Alpine user based on
63 what was configured during instance launch. User data is supported as long
64 as it's a shell script (starts with #!). See the tiny-ec2-bootstrap README
65 for more details. You can still install cloud-init (from the edge testing
66 repositories), but we haven't tested whether it will work correctly for this
67 AMI. If full cloud-init support is important to you please file a bug
68 against this project.
69
70- CloudFormation support is still forthcoming. This requires patches and
71 packaging for the upstream cfn tools that have not yet been accepted.
72 Eventually full CloudFormation support will be available.
diff --git a/alpine-ami.yaml b/alpine-ami.yaml
deleted file mode 100644
index 1505055..0000000
--- a/alpine-ami.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
1variables:
2
3 # NOTE: Configuration is done with a 'variables.yaml' file. If it doesn't
4 # exist, default configuration is copied from 'variables.yaml-default'.
5
6 # NOTE: Changing arch/version/release may require modifying 'make_ami.sh'.
7 arch: x86_64
8 version: "3.9"
9 release: "3.9.3"
10 revision: ""
11
12builders:
13 - type: "amazon-ebssurrogate"
14
15 ### Builder Instance Details
16
17 region: "{{user `region`}}"
18 subnet_id: "{{user `subnet`}}"
19 security_group_id: "{{user `security_group`}}"
20 instance_type: "t3.nano"
21 associate_public_ip_address: "{{user `public_ip`}}"
22 launch_block_device_mappings:
23 - volume_type: "gp2"
24 device_name: "/dev/xvdf"
25 delete_on_termination: "true"
26 volume_size: "{{user `volume_size`}}"
27 ssh_username: "ec2-user"
28 source_ami_filter:
29 # use the latest Amazon Linux AMI
30 filters:
31 virtualization-type: "hvm"
32 root-device-type: "ebs"
33 architecture: "x86_64"
34 name: "amzn2-ami-hvm-2.0.*-gp2"
35 owners:
36 - "137112412989"
37 most_recent: "true"
38
39 ### AMI Build Details
40
41 ami_name: "{{user `ami_name_prefix`}}{{user `release`}}{{user `revision`}}-{{user `arch`}}{{user `ami_name_suffix`}}"
42 ami_description: "{{user `ami_desc_prefix`}}{{user `release`}}{{user `revision`}} {{user `arch`}}{{user `ami_desc_suffix`}}"
43 ami_virtualization_type: "hvm"
44 ami_root_device:
45 source_device_name: "/dev/xvdf"
46 device_name: "/dev/xvda"
47 delete_on_termination: "true"
48 volume_size: "{{user `volume_size`}}"
49 volume_type: "gp2"
50 encrypt_boot: "{{user `encrypt_ami`}}"
51 ena_support: "true"
52 sriov_support: "true"
53 ami_groups: "{{user `ami_access`}}"
54 ami_regions: "{{user `deploy_regions`}}"
55
56
57provisioners:
58 - type: "file"
59 source: "nvme/"
60 destination: "/tmp"
61 - type: "shell"
62 script: "make_ami.sh"
63 environment_vars:
64 - "VERSION={{user `version`}}"
65 - "RELEASE={{user `release`}}"
66 - "REVISION={{user `revision`}}"
67 - "ADD_REPOS='{{user `add_repos`}}'"
68 - "ADD_PKGS='{{user `add_pkgs`}}'"
69 - "ADD_SVCS='{{user `add_svcs`}}'"
70 execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }}"'
diff --git a/gen-readme.py.in b/gen-readme.py.in
deleted file mode 100644
index 84e6ed4..0000000
--- a/gen-readme.py.in
+++ /dev/null
@@ -1,17 +0,0 @@
1@PYTHON@
2
3import yaml
4
5URI_TEMPLATE = "https://{region}.console.aws.amazon.com/ec2/home#launchAmi={ami}"
6ROW_TEMPLATE = "| {release} | {region} | [{ami}]({uri}) |"
7
8
9with open("release.yaml") as fp:
10 releases = yaml.full_load(fp)
11
12for metadata in releases.values():
13 release = str(metadata["alpine-release"])
14
15 for region, ami in metadata["region-identifiers"].items():
16 uri = URI_TEMPLATE.format(**locals())
17 print(ROW_TEMPLATE.format(**locals()))
diff --git a/make_ami.sh b/make_ami.sh
deleted file mode 100755
index d6c65f7..0000000
--- a/make_ami.sh
+++ /dev/null
@@ -1,368 +0,0 @@
1#!/bin/sh
2# vim: set ts=4 et:
3
4set -eu
5
6MIN_VERSION="3.9"
7MIN_RELEASE="3.9.0"
8
9: ${VERSION:="${MIN_VERSION}"} # unless otherwise specified
10: ${RELEASE:="${MIN_RELEASE}"} # unless otherwise specified
11
12: ${APK_TOOLS_URI:="https://github.com/alpinelinux/apk-tools/releases/download/v2.10.3/apk-tools-2.10.3-x86_64-linux.tar.gz"}
13: ${APK_TOOLS_SHA256:="4d0b2cda606720624589e6171c374ec6d138867e03576d9f518dddde85c33839"}
14: ${ALPINE_KEYS:="http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/alpine-keys-2.1-r1.apk"}
15: ${ALPINE_KEYS_SHA256:="9c7bc5d2e24c36982da7aa49b3cfcb8d13b20f7a03720f25625fa821225f5fbc"}
16
17die() {
18 printf '\033[1;31mERROR:\033[0m %s\n' "$@" >&2 # bold red
19 exit 1
20}
21
22einfo() {
23 printf '\n\033[1;36m> %s\033[0m\n' "$@" >&2 # bold cyan
24}
25
26rc_add() {
27 local target="$1"; shift # target directory
28 local runlevel="$1"; shift # runlevel name
29 local services="$*" # names of services
30
31 local svc; for svc in $services; do
32 mkdir -p "$target"/etc/runlevels/$runlevel
33 ln -s /etc/init.d/$svc "$target"/etc/runlevels/$runlevel/$svc
34 echo " * service $svc added to runlevel $runlevel"
35 done
36}
37
38wgets() (
39 local url="$1" # url to fetch
40 local sha256="$2" # expected SHA256 sum of output
41 local dest="$3" # output path and filename
42
43 wget -T 10 -q -O "$dest" "$url"
44 echo "$sha256 $dest" | sha256sum -c > /dev/null
45)
46
47
48validate_block_device() {
49 local dev="$1" # target directory
50
51 lsblk -P --fs "$dev" >/dev/null 2>&1 || \
52 die "'$dev' is not a valid block device"
53
54 if lsblk -P --fs "$dev" | grep -vq 'FSTYPE=""'; then
55 die "Block device '$dev' is not blank"
56 fi
57}
58
59fetch_apk_tools() {
60 local store="$(mktemp -d)"
61 local tarball="$(basename $APK_TOOLS_URI)"
62
63 wgets "$APK_TOOLS_URI" "$APK_TOOLS_SHA256" "$store/$tarball"
64 tar -C "$store" -xf "$store/$tarball"
65
66 find "$store" -name apk
67}
68
69make_filesystem() {
70 local device="$1" # target device path
71 local target="$2" # mount target
72
73 mkfs.ext4 -O ^64bit "$device"
74 e2label "$device" /
75 mount "$device" "$target"
76}
77
78setup_repositories() {
79 local target="$1" # target directory
80 local add_repos="$2" # extra repo lines, comma separated
81
82 mkdir -p "$target"/etc/apk/keys
83
84 if [ "$VERSION" = 'edge' ]; then
85 cat > "$target"/etc/apk/repositories <<EOF
86http://dl-cdn.alpinelinux.org/alpine/edge/main
87http://dl-cdn.alpinelinux.org/alpine/edge/community
88http://dl-cdn.alpinelinux.org/alpine/edge/testing
89EOF
90 else
91 cat > "$target"/etc/apk/repositories <<EOF
92http://dl-cdn.alpinelinux.org/alpine/v$VERSION/main
93http://dl-cdn.alpinelinux.org/alpine/v$VERSION/community
94EOF
95 fi
96
97 echo "$add_repos" | tr , "\012" >> "$target"/etc/apk/repositories
98}
99
100fetch_keys() {
101 local target="$1"
102 local tmp="$(mktemp -d)"
103
104 wgets "$ALPINE_KEYS" "$ALPINE_KEYS_SHA256" "$tmp/alpine-keys.apk"
105 tar -C "$target" -xvf "$tmp"/alpine-keys.apk etc/apk/keys
106 rm -rf "$tmp"
107}
108
109install_base() {
110 local target="$1"
111
112 $apk add --root "$target" --no-cache --initdb alpine-base
113 # verify release matches
114 if [ "$VERSION" != "edge" ]; then
115 ALPINE_RELEASE=$(cat "$target/etc/alpine-release")
116 [ "$RELEASE" = "$ALPINE_RELEASE" ] || \
117 die "Current Alpine $VERSION release ($ALPINE_RELEASE) does not match build ($RELEASE)"
118 fi
119}
120
121setup_chroot() {
122 local target="$1"
123
124 mount -t proc none "$target"/proc
125 mount --bind /dev "$target"/dev
126 mount --bind /sys "$target"/sys
127
128 # Don't want to ship this but it's needed for bootstrap. Will be removed in
129 # the cleanup stage.
130 install -Dm644 /etc/resolv.conf "$target"/etc/resolv.conf
131}
132
133install_core_packages() {
134 local target="$1" # target directory
135 local add_pkgs="$2" # extra packages, space separated
136
137 # Most from: https://git.alpinelinux.org/cgit/alpine-iso/tree/alpine-virt.packages
138 #
139 # sudo - to allow alpine user to become root, disallow root SSH logins
140 # tiny-ec2-bootstrap - to bootstrap system from EC2 metadata
141 #
142 chroot "$target" apk --no-cache add \
143 linux-virt \
144 alpine-mirrors \
145 chrony \
146 haveged \
147 nvme-cli \
148 openssh \
149 sudo \
150 tiny-ec2-bootstrap \
151 tzdata \
152 $(echo "$add_pkgs" | tr , ' ')
153
154 chroot "$target" apk --no-cache add --no-scripts syslinux
155
156 # Disable starting getty for physical ttys because they're all inaccessible
157 # anyhow. With this configuration boot messages will still display in the
158 # EC2 console.
159 sed -Ei '/^tty[0-9]/s/^/#/' \
160 "$target"/etc/inittab
161
162 # Make it a little more obvious who is logged in by adding username to the
163 # prompt
164 sed -i "s/^export PS1='/&\\\\u@/" "$target"/etc/profile
165}
166
167setup_mdev() {
168 local target="$1"
169
170 cp /tmp/nvme-ebs-links "$target"/lib/mdev
171 sed -n -i -e '/# fallback/r /tmp/nvme-ebs-mdev.conf' -e 1x -e '2,${x;p}' -e '${x;p}' "$target"/etc/mdev.conf
172}
173
174create_initfs() {
175 local target="$1"
176
177 # Create ENA feature for mkinitfs
178 echo "kernel/drivers/net/ethernet/amazon" > \
179 "$target"/etc/mkinitfs/features.d/ena.modules
180
181 # Enable ENA and NVME features these don't hurt for any instance and are
182 # hard requirements of the 5 series and i3 series of instances
183 sed -Ei 's/^features="([^"]+)"/features="\1 nvme ena"/' \
184 "$target"/etc/mkinitfs/mkinitfs.conf
185
186 chroot "$target" /sbin/mkinitfs $(basename $(find "$target"/lib/modules/* -maxdepth 0))
187}
188
189setup_extlinux() {
190 local target="$1"
191
192 # Must use disk labels instead of UUID or devices paths so that this works
193 # across instance familes. UUID works for many instances but breaks on the
194 # NVME ones because EBS volumes are hidden behind NVME devices.
195 #
196 # Enable ext4 because the root device is formatted ext4
197 #
198 # Shorten timeout because EC2 has no way to interact with instance console
199 #
200 # ttyS0 is the target for EC2s "Get System Log" feature whereas tty0 is the
201 # target for EC2s "Get Instance Screenshot" feature. Enabling the serial
202 # port early in extlinux gives the most complete output in the system log.
203 sed -Ei -e "s|^[# ]*(root)=.*|\1=LABEL=/|" \
204 -e "s|^[# ]*(default_kernel_opts)=.*|\1=\"console=ttyS0 console=tty0\"|" \
205 -e "s|^[# ]*(serial_port)=.*|\1=ttyS0|" \
206 -e "s|^[# ]*(modules)=.*|\1=sd-mod,usb-storage,ext4|" \
207 -e "s|^[# ]*(default)=.*|\1=virt|" \
208 -e "s|^[# ]*(timeout)=.*|\1=1|" \
209 "$target"/etc/update-extlinux.conf
210}
211
212install_extlinux() {
213 local target="$1"
214
215 chroot "$target" /sbin/extlinux --install /boot
216 chroot "$target" /sbin/update-extlinux --warn-only
217}
218
219setup_fstab() {
220 local target="$1"
221
222 cat > "$target"/etc/fstab <<EOF
223# <fs> <mountpoint> <type> <opts> <dump/pass>
224LABEL=/ / ext4 defaults,noatime 1 1
225EOF
226}
227
228setup_networking() {
229 local target="$1"
230
231 cat > "$target"/etc/network/interfaces <<EOF
232auto lo
233iface lo inet loopback
234
235auto eth0
236iface eth0 inet dhcp
237EOF
238}
239
240enable_services() {
241 local target="$1"
242 local add_svcs="$2"
243
244 rc_add "$target" default chronyd networking sshd tiny-ec2-bootstrap
245 rc_add "$target" sysinit devfs dmesg hwdrivers mdev
246 rc_add "$target" boot acpid bootmisc haveged hostname hwclock modules swap sysctl syslog
247 rc_add "$target" shutdown killprocs mount-ro savecache
248
249 if [ -n "$add_svcs" ]; then
250 local lvl_svcs; for lvl_svcs in $(echo "$add_svcs" | tr : ' '); do
251 rc_add "$target" $(echo "$lvl_svcs" | tr =, ' ')
252 done
253 fi
254}
255
256create_alpine_user() {
257 local target="$1"
258
259 # Allow members of the wheel group to sudo without a password. By default
260 # this will only be the alpine user. This allows us to ship an AMI that is
261 # accessible via SSH using the user's configured SSH keys (thanks to
262 # tiny-ec2-bootstrap) but does not allow remote root access which is the
263 # best-practice.
264 sed -i '/%wheel .* NOPASSWD: .*/s/^# //' "$target"/etc/sudoers
265
266 # There is no real standard ec2 username across AMIs, Amazon uses ec2-user
267 # for their Amazon Linux AMIs but Ubuntu uses ubuntu, Fedora uses fedora,
268 # etc... (see: https://alestic.com/2014/01/ec2-ssh-username/). So our user
269 # and group are alpine because this is Alpine Linux. On instance bootstrap
270 # the user can create whatever users they want and delete this one.
271 chroot "$target" /usr/sbin/addgroup alpine
272 chroot "$target" /usr/sbin/adduser -h /home/alpine -s /bin/sh -G alpine -D alpine
273 chroot "$target" /usr/sbin/addgroup alpine wheel
274 chroot "$target" /usr/bin/passwd -u alpine
275}
276
277configure_ntp() {
278 local target="$1"
279
280 # EC2 provides an instance-local NTP service syncronized with GPS and
281 # atomic clocks in-region. Prefer this over external NTP hosts when running
282 # in EC2.
283 #
284 # See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html
285 sed -e 's/^pool /server /' \
286 -e 's/pool.ntp.org/169.254.169.123/g' \
287 -i "$target"/etc/chrony/chrony.conf
288}
289
290cleanup() {
291 local target="$1"
292
293 # Sweep cruft out of the image that doesn't need to ship or will be
294 # re-generated when the image boots
295 rm -f \
296 "$target"/var/cache/apk/* \
297 "$target"/etc/resolv.conf \
298 "$target"/root/.ash_history \
299 "$target"/etc/*-
300
301 umount \
302 "$target"/dev \
303 "$target"/proc \
304 "$target"/sys
305
306 umount "$target"
307}
308
309version_sorted() {
310 # falsey if $1 version > $2 version
311 printf "%s\n%s" $1 $2 | sort -VC
312}
313
314main() {
315 [ "$VERSION" != 'edge' ] && {
316 version_sorted $MIN_VERSION $VERSION || die "Minimum Alpine version is '$MIN_RELEASE'"
317 version_sorted $MIN_RELEASE $RELEASE || die "Minimum Alpine release is '$MIN_RELEASE'"
318 }
319
320 local add_repos="$ADD_REPOS"
321 local add_pkgs="$ADD_PKGS"
322 local add_svcs="$ADD_SVCS"
323
324 local device="/dev/xvdf"
325 local target="/mnt/target"
326
327 validate_block_device "$device"
328
329 [ -d "$target" ] || mkdir "$target"
330
331 einfo "Fetching static APK tools"
332 apk="$(fetch_apk_tools)"
333
334 einfo "Creating root filesystem"
335 make_filesystem "$device" "$target"
336
337 einfo "Configuring Alpine repositories"
338 setup_repositories "$target" "$add_repos"
339
340 einfo "Fetching Alpine signing keys"
341 fetch_keys "$target"
342
343 einfo "Installing base system"
344 install_base "$target"
345
346 setup_chroot "$target"
347
348 einfo "Installing core packages"
349 install_core_packages "$target" "$add_pkgs"
350
351 einfo "Configuring and enabling boot loader"
352 create_initfs "$target"
353 setup_extlinux "$target"
354 install_extlinux "$target"
355
356 einfo "Configuring system"
357 setup_mdev "$target"
358 setup_fstab "$target"
359 setup_networking "$target"
360 enable_services "$target" "$add_svcs"
361 create_alpine_user "$target"
362 configure_ntp "$target"
363
364 einfo "All done, cleaning up"
365 cleanup "$target"
366}
367
368main "$@"
diff --git a/packer.conf b/packer.conf
new file mode 100644
index 0000000..cfe3086
--- /dev/null
+++ b/packer.conf
@@ -0,0 +1,108 @@
1# This Packer config file is in HOCON, and is converted to JSON at build time.
2# https://github.com/lightbend/config/blob/master/HOCON.md
3# vim: ts=2 et:
4
5builders = [
6 {
7 type = "amazon-ebssurrogate"
8
9 ### Builder Instance Details
10
11 region = "{{user `build_region`}}"
12 subnet_id = "{{user `build_subnet`}}"
13 instance_type = "{{user `build_instance_type`}}"
14 associate_public_ip_address = "{{user `build_public_ip`}}"
15 source_ami_filter {
16 # use the latest Amazon Linux AMI
17 owners = [ "{{user `build_ami_owner`}}" ]
18 most_recent = "{{user `build_ami_latest`}}"
19 filters {
20 virtualization-type = "hvm"
21 root-device-type = "ebs"
22 architecture = "{{user `build_arch`}}"
23 name = "{{user `build_ami_name`}}"
24 }
25 }
26 launch_block_device_mappings = [
27 {
28 volume_type = "gp2"
29 device_name = "/dev/xvdf"
30 delete_on_termination = "true"
31 volume_size = "{{user `ami_volume_size`}}"
32 }
33 ]
34 shutdown_behavior = "terminate"
35 ssh_username = "{{user `build_user`}}"
36
37 ### AMI Build Details
38
39 ami_name = "{{user `ami_name`}}"
40 ami_description = "{{user `ami_desc`}}"
41 tags {
42 Name = "{{user `ami_name`}}"
43 }
44 ami_virtualization_type = "hvm"
45 ami_architecture = "{{user `build_arch`}}" # need packer 1.4.1
46 ami_root_device {
47 volume_type = "gp2"
48 source_device_name = "/dev/xvdf"
49 device_name = "/dev/xvda"
50 delete_on_termination = "true"
51 volume_size = "{{user `ami_volume_size`}}"
52 }
53 encrypt_boot = "{{user `ami_encrypt`}}"
54 ena_support = "true"
55 sriov_support = "true"
56 ami_groups = "{{user `ami_access`}}"
57 ami_regions = "{{user `ami_regions`}}"
58 }
59]
60
61
62provisioners = [
63 {
64 type = "file"
65 source = "nvme/"
66 destination = "/tmp"
67 }
68 {
69 type = "shell"
70 script = "setup-ami"
71 environment_vars = [
72 "VERSION={{user `version`}}"
73 "RELEASE={{user `release`}}"
74 "REVISION={{user `revision`}}"
75 "ARCH={{user `arch`}}"
76 "APK_TOOLS={{user `apk_tools`}}"
77 "APK_TOOLS_SHA256={{user `apk_tools_sha256`}}"
78 "ALPINE_KEYS={{user `alpine_keys`}}"
79 "ALPINE_KEYS_SHA256={{user `alpine_keys_sha256`}}"
80 "REPOS={{user `repos`}}"
81 "PKGS={{user `pkgs`}}"
82 "SVCS={{user `svcs`}}"
83 "KERNEL_MODS={{user `kernel_modules`}}"
84 "KERNEL_OPTS={{user `kernel_options`}}"
85 ]
86 use_env_var_file = "true"
87 execute_command = "sudo sh -c '. {{.EnvVarFile}} && {{.Path}}'"
88 }
89]
90
91
92post-processors = [
93 {
94 type = "manifest"
95 output = "profile/{{user `profile`}}/{{user `profile_build`}}/manifest.json"
96 custom_data {
97 ami_name = "{{user `ami_name`}}"
98 ami_desc = "{{user `ami_desc`}}"
99 profile = "{{user `profile`}}"
100 profile_build = "{{user `profile_build`}}"
101 version = "{{user `version`}}"
102 release = "{{user `release`}}"
103 arch = "{{user `arch`}}"
104 revision = "{{user `revision`}}"
105 end_of_life = "{{user `end_of_life`}}"
106 }
107 }
108]
diff --git a/profiles/README.md b/profiles/README.md
new file mode 100644
index 0000000..107b59e
--- /dev/null
+++ b/profiles/README.md
@@ -0,0 +1,157 @@
1# Profiles
2
3Profiles are collections of related build definitions, which are used to
4generate the `variables.yaml` files that [Packer](https://packer.io) consumes
5when building AMIs.
6
7Profiles use [HOCON](https://github.com/lightbend/config/blob/master/HOCON.md)
8(Human-Optimized Config Object Notation) which allows importing common configs
9from other files, simple variable interpolation, and easy merging of objects.
10This flexibility helps keep configuration for related build targets
11[DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself).
12
13## Core Profiles
14
15Core profile configurations are found in the `base`, `version`, and `arch`
16subdirectories. Core profiles do not have a `.conf` suffix because they're not
17meant to be directly used like target profiles with `make`.
18
19Base core profiles define all build vars with default values -- those left
20empty or null are usually set in version, arch, or target profile configs.
21Base profiles are included in version profiles, and do not need to be included
22in target profiles.
23
24Version core profiles expand on the base profile they include, and set the
25`version`, `release`, `end_of_life` (if known), and the associated Alpine Linux
26`repos`.
27
28Arch core profiles further define architecture-specific variables, such as
29which `apk-tools` and `alpine-keys` to use (and their SHA256 checksums).
30
31## Target Profiles
32
33Target profiles, defined in this directory, are the top-level configuration
34used with `make PROFILE=<profile>`; they must have a `.conf` suffix. Several
35configuration objects are defined and later merged within the `BUILDS` object,
36ultimately defining each individual build.
37
38Simple profiles have an object that loads a "version" core profile and
39another that loads an "arch" core profile. A more complicated version-arch
40matrix profile would have an object for each version and arch.
41
42Additionally, there are one or more objects that define profile-specific
43settings.
44
45The `BUILDS` object's elements merge core and profile configs (with optional
46inline build settings) into named build definitions; these build names can be
47used to specify a subset of a profile's builds:
48`make PROFILE=<profile> BUILDS="<build> ..."`
49
50**Please note that merge order matters!** The merge sequence is version -->
51architecture --> profile --> build.
52
53## Customization
54
55The most important variables to set in your custom profile is `build_region`
56and `build_subnet`. Without these, Packer will not know where to build.
57
58`version` and `release` are meant to match Alpine; however,`revision` can be
59used to track changes to profile or situations where the AMIs needed to be
60rebuilt. The "edge" core version profile sets `revision` to the current
61datetime, otherwise the default is `r0`.
62
63You will probably want to personalize the name and description of your AMI.
64Set `ami_name_prefix` and `ami_name_suffix`; setting `ami_desc_suffix` and
65`ami_desc_suffix` is optional.
66
67Set `build_instance_type` if you want/need to use a different instance type to
68build the image; the default is `t3.nano`.
69
70If 1 GiB is not enough to install the packages in your base AMI, you can set
71the `ami_volume_size` to the number of GiB you need. Note, however, that the
72[tiny-ec2-bootstrap](https://github.com/mcrute/tiny-ec2-bootstrap) init script
73will expand the root partition to use the instance's entire EBS root volume
74during the first boot, so you shouldn't need to make space for anything other
75than installed packages.
76
77Set `ami_encrypt` to "true" to create an encrypted AMI image. Launching images
78from an encrypted AMI results in an encrypted EBS root volume.
79
80To copy newly built AMIs to regions other than the `build_region` region, set
81`ami_regions`. This variable is a *hash*, which allows for finer control over
82inherited values when merging configs. Region identifiers are the keys, a
83value of `true` means the AMI should be copied to that region; `null` or
84`false` indicate that it shouldn't be copied to that region. If you want to
85ensure that the `ami_regions` hash does not inherit any values, set it to
86`null` before configuring your regions. For example:
87```
88ami_regions = null # don't inherit any previous values
89ami_regions {
90 us-west-2 = true
91 eu-north-1 = true
92}
93```
94
95Controlling what packages are installed and enabled in the AMI is the number
96one reason for creating custom profile. The `repos`, `pkgs`, and `svcs` hash
97variables serve precisely that purpose. With some exceptions (noted below),
98they work the same as the `ami_regions` hash: `true` values enable, `false`
99and `null` values disable, and inherited values can be cleared by first setting
100the variable itself to `null`.
101
102With `repos`, the keys are double-quoted URLs to the `apk` repos that you want
103set up; these are initially set in the "version" core profiles. In addition
104to the `true`, `false`, and `null` values, you can also use a "repo alias"
105string value, allowing you to pin packages to be sourced from that particular
106repo. For example, with a profile based from a non-edge core profile, you may
107want to be able to pull packages from the edge testing repo:
108```
109repos {
110 "http://dl-cdn.alpinelinux.org/alpine/edge/testing" = "edge-testing"
111}
112```
113
114The `pkgs` hash's default is set in the base core profile; its keys are
115simply the Alpine package to install (or not install, if the value is `false`
116or `null`). A `true` value installs the package from the default repos; if the
117value is a repo alias string, the package will be pinned to explicitly install
118from that repo. For example:
119```
120pkgs {
121 # install docker-compose from edge-testing repo
122 docker-compose = "edge-testing"
123}
124```
125
126To control when (or whether) a system service starts, use the `svcs` hash
127variable. Its keys are the service names, as they appear in `/etc/init.d`;
128default values are set in the base core profile. Like the other hash
129variables, setting `false` or `null` disable the service, `true` will enable
130the service at the "default" runlevel. The service can be enabled at a
131different runlevel by using that runlevel as the value.
132
133By default, the AMIs built are accessible only by the owning account. To
134make your AMIs publicly available, set the `ami_access` hash variable:
135```
136ami_access {
137 all = true
138}
139```
140
141## Limitations and Caveats
142
143* Hash variables that are reset to clear inherited values *must* be
144 re-defined as a hash, even if it is to remain empty:
145 ```
146 hash_var = null # drops inherited values
147 hash_var {} # re-defines as an empty hash
148 ```
149
150* The AMI's login user is currently hard coded to be `alpine`. Changes to
151 [tiny-ec2-bootstrap](https://github.com/mcrute/tiny-ec2-bootstrap) are
152 required before we can truly make `ami_user` configurable.
153
154* Currently, it is not possible to add/modify/remove arbitrary files (such as
155 service config files) on the filesystem which ultimately becomes the AMI.
156 One workaround is to use a "user data" script to make any necessary changes
157 (during the "default" runlevel) when an instance first launches.
diff --git a/profiles/alpine.conf b/profiles/alpine.conf
new file mode 100644
index 0000000..3727753
--- /dev/null
+++ b/profiles/alpine.conf
@@ -0,0 +1,47 @@
1### Profile for Building the Publically-Available Alpine Linux AMIs
2# vim: ts=2 et:
3
4version-current { include required("version/current") }
5version-edge { include required("version/edge") }
6arch-x86_64 { include required("arch/x86_64") }
7
8# profile vars
9alpine {
10 # default profile revision is 'r0', reset for each new version release!
11 #revision = "r0"
12
13 ami_desc_suffix = " - https://github.com/mcrute/alpine-ec2-ami"
14
15 build_region = "us-west-2"
16 build_subnet = "subnet-b80c36e2"
17 ami_access {
18 all = true # these AMIs are publicly available
19 }
20 ami_regions {
21 #ap-east-1 = true # needs to be enabled first
22 ap-northeast-1 = true
23 ap-northeast-2 = true
24 #ap-northeast-3 = false # available by subscription only
25 ap-southeast-1 = true
26 ap-southeast-2 = true
27 ap-south-1 = true
28 ca-central-1 = true
29 eu-central-1 = true
30 eu-north-1 = true
31 eu-west-1 = true
32 eu-west-2 = true
33 eu-west-3 = true
34 sa-east-1 = true
35 us-east-1 = true
36 us-east-2 = true
37 us-west-1 = true
38 us-west-2 = true
39 }
40}
41
42# Build definitions
43BUILDS {
44 # merge version, arch, and profile vars
45 current-x86_64 = ${version-current} ${arch-x86_64} ${alpine}
46 edge-x86_64 = ${version-edge} ${arch-x86_64} ${alpine}
47}
diff --git a/profiles/arch/aarch64 b/profiles/arch/aarch64
new file mode 120000
index 0000000..f1b77e5
--- /dev/null
+++ b/profiles/arch/aarch64
@@ -0,0 +1 @@
aarch64-1 \ No newline at end of file
diff --git a/profiles/arch/aarch64-1 b/profiles/arch/aarch64-1
new file mode 100644
index 0000000..37564e5
--- /dev/null
+++ b/profiles/arch/aarch64-1
@@ -0,0 +1,10 @@
1### aarch64 vars, revision 1
2# vim: ts=2 et:
3
4arch = "aarch64"
5build_arch = "arm64"
6build_instance_type = "a1.medium"
7apk_tools = "https://github.com/alpinelinux/apk-tools/releases/download/v2.10.3/apk-tools-2.10.3-aarch64-linux.tar.gz"
8apk_tools_sha256 = "58a07e547c83c3a30eb0a0bd73db57d6bbaf92cc093df7a1d9805631f7d349e3"
9alpine_keys = "http://dl-cdn.alpinelinux.org/alpine/v3.9/main/aarch64/alpine-keys-2.1-r1.apk"
10alpine_keys_sha256 = "1ae4cebb43adee47a68aa891660e69a1ac6467690daca6f211aabff36a17cad1"
diff --git a/profiles/arch/x86_64 b/profiles/arch/x86_64
new file mode 120000
index 0000000..f3c4f51
--- /dev/null
+++ b/profiles/arch/x86_64
@@ -0,0 +1 @@
x86_64-1 \ No newline at end of file
diff --git a/profiles/arch/x86_64-1 b/profiles/arch/x86_64-1
new file mode 100644
index 0000000..40b60dc
--- /dev/null
+++ b/profiles/arch/x86_64-1
@@ -0,0 +1,9 @@
1### x86_64 vars, revision 1
2# vim: ts=2 et:
3
4arch = "x86_64"
5build_arch = "x86_64"
6apk_tools = "https://github.com/alpinelinux/apk-tools/releases/download/v2.10.3/apk-tools-2.10.3-x86_64-linux.tar.gz"
7apk_tools_sha256 = "4d0b2cda606720624589e6171c374ec6d138867e03576d9f518dddde85c33839"
8alpine_keys = "http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/alpine-keys-2.1-r1.apk"
9alpine_keys_sha256 = "9c7bc5d2e24c36982da7aa49b3cfcb8d13b20f7a03720f25625fa821225f5fbc"
diff --git a/profiles/base/1 b/profiles/base/1
new file mode 100644
index 0000000..fdda3f0
--- /dev/null
+++ b/profiles/base/1
@@ -0,0 +1,87 @@
1### base vars, revision 1
2# vim: ts=2 et:
3
4# Profile/Build
5profile = null
6profile_build = null
7revision = "r0"
8
9# Versioning
10version = null
11release = null
12end_of_life = null
13
14# Architecture
15arch = null
16build_arch = null
17
18# Builder-instance
19build_region = null
20build_subnet = null
21build_instance_type = "t3.nano"
22build_public_ip = null
23build_user = "ec2-user"
24build_ami_name = "amzn2-ami-hvm-2.0.*-gp2"
25build_ami_owner = "137112412989"
26build_ami_latest = "true"
27
28# AMI build/deploy
29ami_name_prefix = "alpine-ami-"
30ami_name_suffix = ""
31ami_desc_prefix = "Alpine Linux "
32ami_desc_suffix = ""
33ami_volume_size = "1"
34ami_encrypt = "false"
35ami_user = "alpine" # modification currently not supported
36ami_access = {}
37ami_regions = {}
38# NOTE: the following are python format strings, resolved in resolve-profile.py
39ami_name = "{var.ami_name_prefix}{var.release}-{var.arch}-{var.revision}{var.ami_name_suffix}"
40ami_desc = "{var.ami_desc_prefix}{var.release} {var.arch} {var.revision}{var.ami_desc_suffix}"
41
42# AMI configuration
43apk_tools = null
44apk_tools_sha256 = null
45alpine_keys = null
46alpine_keys_sha256 = null
47repos {}
48pkgs {
49 linux-virt = true
50 alpine-mirrors = true
51 chrony = true
52 nvme-cli = true
53 openssh = true
54 sudo = true
55 tiny-ec2-bootstrap = true
56 tzdata = true
57}
58svcs {
59 devfs = "sysinit"
60 dmesg = "sysinit"
61 hwdrivers = "sysinit"
62 mdev = "sysinit"
63 acpid = "boot"
64 bootmisc = "boot"
65 hostname = "boot"
66 hwclock = "boot"
67 modules = "boot"
68 swap = "boot"
69 sysctl = "boot"
70 syslog = "boot"
71 chronyd = "default"
72 networking = "default"
73 sshd = "default"
74 tiny-ec2-bootstrap = "default"
75 killprocs = "shutdown"
76 mount-ro = "shutdown"
77 savecache = "shutdown"
78}
79kernel_modules {
80 sd-mod = true
81 usb-storage = true
82 ext4 = true
83}
84kernel_options {
85 "console=ttyS0" = true
86 "console=tty0" = true
87}
diff --git a/profiles/base/current b/profiles/base/current
new file mode 120000
index 0000000..56a6051
--- /dev/null
+++ b/profiles/base/current
@@ -0,0 +1 @@
1 \ No newline at end of file
diff --git a/profiles/test.conf b/profiles/test.conf
new file mode 100644
index 0000000..eb7f2a3
--- /dev/null
+++ b/profiles/test.conf
@@ -0,0 +1,31 @@
1### Profile for Testing Builds
2# vim: ts=2 et:
3
4version-current { include required("version/current") }
5version-edge { include required("version/edge") }
6arch-x86_64 { include required("arch/x86_64") }
7arch-aarch64 { include required("arch/aarch64") }
8
9# specific to this profile's builds
10test {
11 # default revision is 'r0', recomment/reset for each new version release!
12 #revision = "r0"
13
14 ami_name_prefix = "test-"
15 ami_desc_prefix = "Alpine Test "
16 build_region = "us-west-2"
17 build_subnet = "subnet-033a30d7b5220d177"
18}
19
20# Build definitions
21BUILDS {
22 # merge version, arch, profile, and build vars
23 current-x86_64 = ${version-current} ${arch-x86_64} ${test}
24 edge-x86_64 = ${version-edge} ${arch-x86_64} ${test}
25
26 # aarch64 AMI builds are under development
27 edge-aarch64 = ${version-edge} ${arch-aarch64} ${test} {
28 # other subnet doesn't do a1.* instances
29 build_subnet = "subnet-08dfc622745f7d96a"
30 }
31}
diff --git a/profiles/version/3.9 b/profiles/version/3.9
new file mode 100644
index 0000000..a08c079
--- /dev/null
+++ b/profiles/version/3.9
@@ -0,0 +1,14 @@
1### version 3.9 vars
2# vim: ts=2 et:
3
4# start with base vars
5include required("../base/current")
6
7# set version-specific vars
8version = "3.9"
9release = "3.9.4"
10end_of_life = "2021-01-01"
11repos {
12 "http://dl-cdn.alpinelinux.org/alpine/v3.9/main" = true
13 "http://dl-cdn.alpinelinux.org/alpine/v3.9/community" = true
14}
diff --git a/profiles/version/current b/profiles/version/current
new file mode 120000
index 0000000..a02597f
--- /dev/null
+++ b/profiles/version/current
@@ -0,0 +1 @@
3.9 \ No newline at end of file
diff --git a/profiles/version/edge b/profiles/version/edge
new file mode 100644
index 0000000..07a04b0
--- /dev/null
+++ b/profiles/version/edge
@@ -0,0 +1,18 @@
1### edge vars
2# vim: ts=2 et:
3
4# based on current
5include required("current")
6
7# add edge-specific tweaks...
8version = "edge"
9release = "edge"
10end_of_life = "@TOMORROW@"
11revision = "@NOW@"
12
13repos = null # remove all values from 'current'
14repos {
15 "http://dl-cdn.alpinelinux.org/alpine/edge/main" = true
16 "http://dl-cdn.alpinelinux.org/alpine/edge/community" = true
17 "http://dl-cdn.alpinelinux.org/alpine/edge/testing" = true
18}
diff --git a/release.yaml b/release.yaml
deleted file mode 100644
index f97beb3..0000000
--- a/release.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
1alpine-ami-3.9.3-x86_64:
2 description: "Alpine Linux 3.9.3 x86_64"
3 alpine-release: 3.9.3
4 kernel-flavor: virt
5 ami-release-date: "2019-03-03 01:03:41"
6 region-identifiers:
7 ap-northeast-1: ami-001e74131496d0212
8 ap-northeast-2: ami-09a26b03424d75667
9 ap-south-1: ami-03534f64f8b87aafc
10 ap-southeast-1: ami-0d5f2950efcd55b0e
11 ap-southeast-2: ami-0660edcba4ba7c8a0
12 ca-central-1: ami-0bf4ea1f0f86283bb
13 eu-central-1: ami-060d9bbde8d5047e8
14 eu-north-1: ami-0a5284750fcf11d18
15 eu-west-1: ami-0af60b964eb2f09d3
16 eu-west-2: ami-097405edd3790cf8b
17 eu-west-3: ami-0078916a37514bb9a
18 sa-east-1: ami-09e0025e60328ea6d
19 us-east-1: ami-05c8c48601c2303af
20 us-east-2: ami-064d64386a89de1e6
21 us-west-1: ami-04a4711d62db12ba0
22 us-west-2: ami-0ff56870cf29d4f02
diff --git a/releases/README.md b/releases/README.md
new file mode 100644
index 0000000..2121038
--- /dev/null
+++ b/releases/README.md
@@ -0,0 +1,77 @@
1# Alpine Linux EC2 AMIs
2
3**These are not official AWS or Alpine images. They are community built and
4supported.**
5
6These AMIs should work with most EC2 features such as Elastic Network Adapters
7and NVMe EBS volumes. If you find any problems launching them on current
8generation instances, please open an [issue](https://github.com/mcrute/alpine-ec2-ami/issues)
9and include as much detailed information as possible.
10
11During the *first boot* of instances created with these AMIs, the lightweight
12[tiny-ec2-bootstrap](https://github.com/mcrute/tiny-ec2-bootstrap) init
13script...
14- sets the instance's hostname,
15- installs the SSH authorized_keys for the 'alpine' user,
16- disables 'root' and 'alpine' users' passwords,
17- expands the root partition to use all available EBS volume space,
18- and executes a "user data" script (must be a shell script that starts with `#!`)
19
20If you launch these AMIs to build other images (via [Packer](https://packer.io),
21etc.), don't forget to remove `/var/lib/cloud/.bootstrap-complete` --
22otherwise, instances launched from those second-generation AMIs will not run
23`tiny-ec2-bootstrap` on their first boot.
24
25The more popular [cloud-init](https://cloudinit.readthedocs.io/en/latest/)
26is currently not supported on Alpine Linux. If `cloud-init` support is
27important to you, please open an [issue](https://github.com/mcrute/alpine-ec2-ami/issues).
28
29## AMIs
30
31### Alpine Linux 3.9.4 (2019-05-28)
32<details><summary><i>click to show/hide</i></summary><p>
33
34| Region | alpine-ami-3.9.4-x86_64-r0 |
35| ------ | --- |
36| ap-northeast-1 | [ami-0251fa7f8f8ed0a3b](https://ap-northeast-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0251fa7f8f8ed0a3b) ([launch](https://ap-northeast-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0251fa7f8f8ed0a3b)) |
37| ap-northeast-2 | [ami-0bb32f18ed247323e](https://ap-northeast-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0bb32f18ed247323e) ([launch](https://ap-northeast-2.console.aws.amazon.com/ec2/home#launchAmi=ami-0bb32f18ed247323e)) |
38| ap-south-1 | [ami-0ca42c8d33ec3ef66](https://ap-south-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0ca42c8d33ec3ef66) ([launch](https://ap-south-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0ca42c8d33ec3ef66)) |
39| ap-southeast-1 | [ami-032330b6de2f39f75](https://ap-southeast-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-032330b6de2f39f75) ([launch](https://ap-southeast-1.console.aws.amazon.com/ec2/home#launchAmi=ami-032330b6de2f39f75)) |
40| ap-southeast-2 | [ami-0681743c5235cb677](https://ap-southeast-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0681743c5235cb677) ([launch](https://ap-southeast-2.console.aws.amazon.com/ec2/home#launchAmi=ami-0681743c5235cb677)) |
41| ca-central-1 | [ami-0dfcf967a696ee901](https://ca-central-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0dfcf967a696ee901) ([launch](https://ca-central-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0dfcf967a696ee901)) |
42| eu-central-1 | [ami-07a8060b90f208cf2](https://eu-central-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-07a8060b90f208cf2) ([launch](https://eu-central-1.console.aws.amazon.com/ec2/home#launchAmi=ami-07a8060b90f208cf2)) |
43| eu-north-1 | [ami-0f25dd1f2ab208b34](https://eu-north-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0f25dd1f2ab208b34) ([launch](https://eu-north-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0f25dd1f2ab208b34)) |
44| eu-west-1 | [ami-07453094c6d42a07e](https://eu-west-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-07453094c6d42a07e) ([launch](https://eu-west-1.console.aws.amazon.com/ec2/home#launchAmi=ami-07453094c6d42a07e)) |
45| eu-west-2 | [ami-03fa8e7cff9293332](https://eu-west-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-03fa8e7cff9293332) ([launch](https://eu-west-2.console.aws.amazon.com/ec2/home#launchAmi=ami-03fa8e7cff9293332)) |
46| eu-west-3 | [ami-07aad42fdc4a7e79b](https://eu-west-3.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-07aad42fdc4a7e79b) ([launch](https://eu-west-3.console.aws.amazon.com/ec2/home#launchAmi=ami-07aad42fdc4a7e79b)) |
47| sa-east-1 | [ami-04cac088d12e5ebf0](https://sa-east-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-04cac088d12e5ebf0) ([launch](https://sa-east-1.console.aws.amazon.com/ec2/home#launchAmi=ami-04cac088d12e5ebf0)) |
48| us-east-1 | [ami-0c2c618b193741157](https://us-east-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0c2c618b193741157) ([launch](https://us-east-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0c2c618b193741157)) |
49| us-east-2 | [ami-012e1a22371695544](https://us-east-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-012e1a22371695544) ([launch](https://us-east-2.console.aws.amazon.com/ec2/home#launchAmi=ami-012e1a22371695544)) |
50| us-west-1 | [ami-00f0f067a7d90b7e4](https://us-west-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-00f0f067a7d90b7e4) ([launch](https://us-west-1.console.aws.amazon.com/ec2/home#launchAmi=ami-00f0f067a7d90b7e4)) |
51| us-west-2 | [ami-0ed0fed8f127914fb](https://us-west-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0ed0fed8f127914fb) ([launch](https://us-west-2.console.aws.amazon.com/ec2/home#launchAmi=ami-0ed0fed8f127914fb)) |
52
53</p></details>
54
55### Alpine Linux Edge (2019-05-28)
56<details><summary><i>click to show/hide</i></summary><p>
57
58| Region | alpine-ami-edge-x86_64-20190528032210 |
59| ------ | --- |
60| ap-northeast-1 | [ami-03a19ed410069a4d8](https://ap-northeast-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-03a19ed410069a4d8) ([launch](https://ap-northeast-1.console.aws.amazon.com/ec2/home#launchAmi=ami-03a19ed410069a4d8)) |
61| ap-northeast-2 | [ami-05988a6c4660792ce](https://ap-northeast-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-05988a6c4660792ce) ([launch](https://ap-northeast-2.console.aws.amazon.com/ec2/home#launchAmi=ami-05988a6c4660792ce)) |
62| ap-south-1 | [ami-08aaeba360cdab5a4](https://ap-south-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-08aaeba360cdab5a4) ([launch](https://ap-south-1.console.aws.amazon.com/ec2/home#launchAmi=ami-08aaeba360cdab5a4)) |
63| ap-southeast-1 | [ami-01ae6c2b20966a358](https://ap-southeast-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-01ae6c2b20966a358) ([launch](https://ap-southeast-1.console.aws.amazon.com/ec2/home#launchAmi=ami-01ae6c2b20966a358)) |
64| ap-southeast-2 | [ami-00193ff2f592dc22c](https://ap-southeast-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-00193ff2f592dc22c) ([launch](https://ap-southeast-2.console.aws.amazon.com/ec2/home#launchAmi=ami-00193ff2f592dc22c)) |
65| ca-central-1 | [ami-086b7f5aa4cf0194e](https://ca-central-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-086b7f5aa4cf0194e) ([launch](https://ca-central-1.console.aws.amazon.com/ec2/home#launchAmi=ami-086b7f5aa4cf0194e)) |
66| eu-central-1 | [ami-089db5b316937779b](https://eu-central-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-089db5b316937779b) ([launch](https://eu-central-1.console.aws.amazon.com/ec2/home#launchAmi=ami-089db5b316937779b)) |
67| eu-north-1 | [ami-02ed2f6e56115d6f2](https://eu-north-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-02ed2f6e56115d6f2) ([launch](https://eu-north-1.console.aws.amazon.com/ec2/home#launchAmi=ami-02ed2f6e56115d6f2)) |
68| eu-west-1 | [ami-0afa00bfa1c870509](https://eu-west-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0afa00bfa1c870509) ([launch](https://eu-west-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0afa00bfa1c870509)) |
69| eu-west-2 | [ami-0b1e309dfd74525f2](https://eu-west-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0b1e309dfd74525f2) ([launch](https://eu-west-2.console.aws.amazon.com/ec2/home#launchAmi=ami-0b1e309dfd74525f2)) |
70| eu-west-3 | [ami-0404d34bb3376e370](https://eu-west-3.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0404d34bb3376e370) ([launch](https://eu-west-3.console.aws.amazon.com/ec2/home#launchAmi=ami-0404d34bb3376e370)) |
71| sa-east-1 | [ami-053be80e8c7b1ad62](https://sa-east-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-053be80e8c7b1ad62) ([launch](https://sa-east-1.console.aws.amazon.com/ec2/home#launchAmi=ami-053be80e8c7b1ad62)) |
72| us-east-1 | [ami-0d1ea89d2b00334f5](https://us-east-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0d1ea89d2b00334f5) ([launch](https://us-east-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0d1ea89d2b00334f5)) |
73| us-east-2 | [ami-0939714c9fe9ec10e](https://us-east-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0939714c9fe9ec10e) ([launch](https://us-east-2.console.aws.amazon.com/ec2/home#launchAmi=ami-0939714c9fe9ec10e)) |
74| us-west-1 | [ami-0b9c5086efa0f067b](https://us-west-1.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0b9c5086efa0f067b) ([launch](https://us-west-1.console.aws.amazon.com/ec2/home#launchAmi=ami-0b9c5086efa0f067b)) |
75| us-west-2 | [ami-0719ffe4d94e67432](https://us-west-2.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId=ami-0719ffe4d94e67432) ([launch](https://us-west-2.console.aws.amazon.com/ec2/home#launchAmi=ami-0719ffe4d94e67432)) |
76
77</p></details>
diff --git a/releases/alpine.yaml b/releases/alpine.yaml
new file mode 100644
index 0000000..9823199
--- /dev/null
+++ b/releases/alpine.yaml
@@ -0,0 +1,58 @@
1current-x86_64:
2 3.9.4:
3 alpine-ami-3.9.4-x86_64-r0:
4 description: Alpine Linux 3.9.4 x86_64 r0 - https://github.com/mcrute/alpine-ec2-ami
5 profile: alpine
6 profile_build: current-x86_64
7 version: '3.9'
8 release: 3.9.4
9 arch: x86_64
10 revision: r0
11 end_of_life: '2021-01-01T00:00:00'
12 build_time: 1559014278
13 artifacts:
14 ap-northeast-1: ami-0251fa7f8f8ed0a3b
15 ap-northeast-2: ami-0bb32f18ed247323e
16 ap-south-1: ami-0ca42c8d33ec3ef66
17 ap-southeast-1: ami-032330b6de2f39f75
18 ap-southeast-2: ami-0681743c5235cb677
19 ca-central-1: ami-0dfcf967a696ee901
20 eu-central-1: ami-07a8060b90f208cf2
21 eu-north-1: ami-0f25dd1f2ab208b34
22 eu-west-1: ami-07453094c6d42a07e
23 eu-west-2: ami-03fa8e7cff9293332
24 eu-west-3: ami-07aad42fdc4a7e79b
25 sa-east-1: ami-04cac088d12e5ebf0
26 us-east-1: ami-0c2c618b193741157
27 us-east-2: ami-012e1a22371695544
28 us-west-1: ami-00f0f067a7d90b7e4
29 us-west-2: ami-0ed0fed8f127914fb
30edge-x86_64:
31 edge:
32 alpine-ami-edge-x86_64-20190528032210:
33 description: Alpine Linux edge x86_64 20190528032210 - https://github.com/mcrute/alpine-ec2-ami
34 profile: alpine
35 profile_build: edge-x86_64
36 version: edge
37 release: edge
38 arch: x86_64
39 revision: '20190528032210'
40 end_of_life: '2019-05-29T03:22:10'
41 build_time: 1559014836
42 artifacts:
43 ap-northeast-1: ami-03a19ed410069a4d8
44 ap-northeast-2: ami-05988a6c4660792ce
45 ap-south-1: ami-08aaeba360cdab5a4
46 ap-southeast-1: ami-01ae6c2b20966a358
47 ap-southeast-2: ami-00193ff2f592dc22c
48 ca-central-1: ami-086b7f5aa4cf0194e
49 eu-central-1: ami-089db5b316937779b
50 eu-north-1: ami-02ed2f6e56115d6f2
51 eu-west-1: ami-0afa00bfa1c870509
52 eu-west-2: ami-0b1e309dfd74525f2
53 eu-west-3: ami-0404d34bb3376e370
54 sa-east-1: ami-053be80e8c7b1ad62
55 us-east-1: ami-0d1ea89d2b00334f5
56 us-east-2: ami-0939714c9fe9ec10e
57 us-west-1: ami-0b9c5086efa0f067b
58 us-west-2: ami-0719ffe4d94e67432
diff --git a/scripts/gen-release-readme.py.in b/scripts/gen-release-readme.py.in
new file mode 100644
index 0000000..764869d
--- /dev/null
+++ b/scripts/gen-release-readme.py.in
@@ -0,0 +1,115 @@
1@PYTHON@
2# vim: ts=4 et:
3
4from datetime import datetime
5from distutils.version import StrictVersion
6import functools
7import os
8import re
9import sys
10import yaml
11
12if len(sys.argv) != 2:
13 sys.exit("Usage: " + os.path.basename(__file__) + "<profile>")
14
15PROFILE = sys.argv[1]
16
17RELEASE_DIR = os.path.join(
18 os.path.dirname(os.path.realpath(__file__)),
19 '..', 'releases'
20)
21
22README_MD = os.path.join( RELEASE_DIR, 'README.md')
23RELEASE_YAML = os.path.join( RELEASE_DIR, PROFILE + '.yaml')
24
25# read in releases/<profile>.yaml
26with open(RELEASE_YAML, 'r') as data:
27 RELEASES = yaml.safe_load(data)
28
29sections = {}
30
31for build, releases in RELEASES.items():
32 for release, amis in releases.items():
33 if release in sections:
34 rel = sections[release]
35 else:
36 rel = {
37 'built': {},
38 'name': {},
39 'ami': {}
40 }
41 for name, info in amis.items():
42 arch = info['arch']
43 built = info['build_time']
44 if (arch not in rel['built'] or
45 rel['built'][arch] < built):
46 rel['name'][arch] = name
47 rel['built'][arch] = built
48 for region, ami in info['artifacts'].items():
49 if region not in rel['ami']:
50 rel['ami'][region] = {}
51 rel['ami'][region][arch] = ami
52 sections[release] = rel
53
54SECTION = """
55### Alpine Linux {release} ({date})
56<details><summary><i>click to show/hide</i></summary><p>
57
58{rows}
59
60</p></details>
61"""
62
63AMI = " [{id}](https://{r}.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId={id}) " + \
64 "([launch](https://{r}.console.aws.amazon.com/ec2/home#launchAmi={id})) |"
65
66ARCHS = ['x86_64', 'aarch64']
67
68
69# most -> least recent version, edge at end
70def ver_cmp(a, b):
71 try:
72 if StrictVersion(a) < StrictVersion(b):
73 return 1
74 if StrictVersion(a) > StrictVersion(b):
75 return -1
76 return 0
77 except ValueError:
78 # "edge" doesn't work with StrictVersion
79 if a == 'edge':
80 return 1
81 if b == 'edge':
82 return -1
83 return 0
84
85
86ami_list = "## AMIs\n"
87
88for release in sorted(list(sections.keys()), key=functools.cmp_to_key(ver_cmp)):
89 info = sections[release]
90 rows = []
91 rows.append('| Region |')
92 rows.append('| ------ |')
93 for arch in ARCHS:
94 if arch in info['name']:
95 rows[0] += ' {n} |'.format(n=info['name'][arch])
96 rows[1] += ' --- |'
97 for region, amis in info['ami'].items():
98 row = '| {r} |'.format(r=region)
99 for arch in ARCHS:
100 if arch in amis:
101 row += AMI.format(r=region, id=amis[arch])
102 rows.append(row)
103 ami_list += SECTION.format(
104 release=release.capitalize(),
105 date=datetime.utcfromtimestamp(max(info['built'].values())).date(),
106 rows="\n".join(rows)
107 )
108
109with open(README_MD, 'r') as file:
110 readme = file.read()
111
112readme_re = re.compile('## AMIs.*\Z', re.S)
113
114with open(README_MD, 'w') as file:
115 file.write(readme_re.sub(ami_list, readme))
diff --git a/scripts/make-amis b/scripts/make-amis
new file mode 100755
index 0000000..d93a266
--- /dev/null
+++ b/scripts/make-amis
@@ -0,0 +1,61 @@
1#!/bin/sh
2# vim: set ts=4 et:
3
4export PACKER=${PACKER:-packer}
5
6cd build || exit 1
7
8# we need a profile, at least
9if [ $# -eq 0 ]; then
10 echo "Usage: $(basename "$0") <profile> [ <build> ... ]" >&2
11 exit 1
12fi
13
14PROFILE=$1; shift
15
16# no build(s) specified? do all the builds!
17[ $# -gt 0 ] && BUILDS="$*" || BUILDS=$(ls "profile/$PROFILE")
18
19for BUILD in $BUILDS
20do
21 printf "\n*** $BUILD ***\n\n"
22
23 BUILD_DIR="profile/$PROFILE/$BUILD"
24
25 # get version, release, and arch
26 eval "$(
27 grep -E '"(version|release|arch)"' "$BUILD_DIR/vars.json" | \
28 sed -e 's/[ ",]//g' -e 's/:/=/g'
29 )"
30
31 if [ "$version" != 'edge' ]; then
32 # get current Alpine release for this version
33 alpine_release=$(
34 curl -s "http://dl-cdn.alpinelinux.org/alpine/v$version/main/$arch/" | \
35 grep '"alpine-base-' | cut -d'"' -f2 | cut -d- -f3
36 )
37 # update core version profile's release if necessary
38 if [ "$alpine_release" != "$release" ]; then
39 printf "=== New release ($alpine_release) detected! ===\n\n"
40 sed -i '' -e "s/$release/$alpine_release/" "../profiles/version/$version"
41 ./resolve-profile.py "$PROFILE"
42 # NOTE: this does NOT update 'revision', it's at target profile/build level
43 fi
44 fi
45
46 # execute packer, capture output and exit code
47 (
48 "$PACKER" build -var-file="$BUILD_DIR/vars.json" packer.json
49 echo $? >"$BUILD_DIR/exit"
50 ) | tee "$BUILD_DIR/output"
51 EXIT=$(cat "$BUILD_DIR/exit")
52
53 if [ "$EXIT" = "0" ]; then
54 ./update-release.py "$PROFILE" "$BUILD"
55 else
56 # unless AMI revision already exists, exit
57 grep -q 'is used by an existing AMI' "$BUILD_DIR/output" || exit "$EXIT"
58 fi
59done
60
61# TODO? if PROFILE = alpine-amis then prune?, gen-releases?
diff --git a/nvme/nvme-ebs-links b/scripts/nvme/nvme-ebs-links
index f2c470b..e418657 100755
--- a/nvme/nvme-ebs-links
+++ b/scripts/nvme/nvme-ebs-links
@@ -2,7 +2,7 @@
2 2
3[ -x /usr/sbin/nvme ] || exit 3[ -x /usr/sbin/nvme ] || exit
4 4
5PROC="$(basename $0)[$$]" 5PROC="$(basename "$0")[$$]"
6 6
7log() { 7log() {
8 FACILITY="kern.$1" 8 FACILITY="kern.$1"
@@ -16,8 +16,8 @@ raw_ebs_alias() {
16 16
17case $ACTION in 17case $ACTION in
18 add|"") 18 add|"")
19 BASE=$(echo $MDEV | sed -re 's/^(nvme[0-9]+n[0-9]+).*/\1/') 19 BASE=$(echo "$MDEV" | sed -re 's/^(nvme[0-9]+n[0-9]+).*/\1/')
20 PART=$(echo $MDEV | sed -re 's/nvme[0-9]+n[0-9]+p?//g') 20 PART=$(echo "$MDEV" | sed -re 's/nvme[0-9]+n[0-9]+p?//g')
21 MAXTRY=50 21 MAXTRY=50
22 TRY=0 22 TRY=0
23 until [ -n "$EBS" ]; do 23 until [ -n "$EBS" ]; do
@@ -30,14 +30,15 @@ case $ACTION in
30 fi 30 fi
31 sleep 0.1 31 sleep 0.1
32 done 32 done
33 EBS=${EBS#/dev/}$PART 33 # remove any leading '/dev/', 'sd', or 'xvd', and append partition
34 ln -sf "$MDEV" "${EBS/xvd/sd}" && log notice "Added ${EBS/xvd/sd} symlink for $MDEV" 34 EBS="${${${EBS#/dev/}#sd}#xvd}$PART"
35 ln -sf "$MDEV" "${EBS/sd/xvd}" && log notice "Added ${EBS/sd/xvd} symlink for $MDEV" 35 ln -sf "$MDEV" "sd$EBS" && log notice "Added sd$EBS symlink for $MDEV"
36 ln -sf "$MDEV" "xvd$EBS" && log notice "Added xvd$EBS symlink for $MDEV"
36 ;; 37 ;;
37 remove) 38 remove)
38 for TARGET in sd* xvd* 39 for TARGET in sd* xvd*
39 do 40 do
40 [ "$(readlink $TARGET 2>/dev/null)" = "$MDEV" ] && rm -f "$TARGET" && log notice "Removed $TARGET symlink for $MDEV" 41 [ "$(readlink "$TARGET" 2>/dev/null)" = "$MDEV" ] && rm -f "$TARGET" && log notice "Removed $TARGET symlink for $MDEV"
41 done 42 done
42 ;; 43 ;;
43esac 44esac
diff --git a/nvme/nvme-ebs-mdev.conf b/scripts/nvme/nvme-ebs-mdev.conf
index c30b6fd..c30b6fd 100644
--- a/nvme/nvme-ebs-mdev.conf
+++ b/scripts/nvme/nvme-ebs-mdev.conf
diff --git a/scripts/prune-amis.py.in b/scripts/prune-amis.py.in
new file mode 100644
index 0000000..dd6f6d8
--- /dev/null
+++ b/scripts/prune-amis.py.in
@@ -0,0 +1,133 @@
1@PYTHON@
2# vim: ts=4 et:
3
4from datetime import datetime
5import os
6import sys
7import boto3
8import yaml
9
10LEVELS = ['revision', 'release', 'version']
11
12if 3 < len(sys.argv) > 4 or sys.argv[1] not in LEVELS:
13 sys.exit("Usage: " + os.path.basename(__file__) + """ <level> <profile> [<build>]
14<level> :-
15 revision - keep only the latest revision per release
16 release - keep only the latest release per version
17 version - keep only the versions that aren't end-of-life""")
18
19NOW = datetime.utcnow()
20LEVEL = sys.argv[1]
21PROFILE = sys.argv[2]
22BUILD = None if len(sys.argv) == 3 else sys.argv[3]
23
24RELEASE_YAML = os.path.join(
25 os.path.dirname(os.path.realpath(__file__)),
26 '..', 'releases', PROFILE + '.yaml'
27)
28
29with open(RELEASE_YAML, 'r') as data:
30 BEFORE = yaml.safe_load(data)
31
32known = {}
33prune = {}
34after = {}
35
36# for all builds in the profile...
37for build_name, releases in BEFORE.items():
38
39 # this is not the build that was specified
40 if BUILD is not None and BUILD != build_name:
41 print('< skipping {0}/{1}'.format(PROFILE, build_name))
42 # ensure its release data remains intact
43 after[build_name] = BEFORE[build_name]
44 continue
45 else:
46 print('> PRUNING {0}/{1} for {2}'.format(PROFILE, build_name, LEVEL))
47
48 criteria = {}
49
50 # scan releases for pruning criteria
51 for release, amis in releases.items():
52 for ami_name, info in amis.items():
53 version = info['version']
54 if info['end_of_life']:
55 eol = datetime.fromisoformat(info['end_of_life'])
56 else:
57 eol = None
58 built = info['build_time']
59 for region, ami_id in info['artifacts'].items():
60 if region not in known:
61 known[region] = []
62 known[region].append(ami_id)
63
64 if LEVEL == 'revision':
65 # find build timestamp of most recent revision, per release
66 if release not in criteria or built > criteria[release]:
67 criteria[release] = built
68 elif LEVEL == 'release':
69 # find build timestamp of most recent revision, per version
70 if version not in criteria or built > criteria[version]:
71 criteria[version] = built
72 elif LEVEL == 'version':
73 # find latest EOL date, per version
74 if (version not in criteria or not criteria[version]) or (
75 eol and eol > criteria[version]):
76 criteria[version] = eol
77
78 # rescan again to determine what doesn't make the cut
79 for release, amis in releases.items():
80 for ami_name, info in amis.items():
81 version = info['version']
82 if info['end_of_life']:
83 eol = datetime.fromisoformat(info['end_of_life'])
84 else:
85 eol = None
86 built = info['build_time']
87 if ((LEVEL == 'revision' and built < criteria[release]) or
88 (LEVEL == 'release' and built < criteria[version]) or
89 (LEVEL == 'version' and criteria[version] and (
90 (version != 'edge' and criteria[version] < NOW) or
91 (version == 'edge' and ((not eol) or (eol < NOW)))
92 ))):
93 for region, ami_id in info['artifacts'].items():
94 if region not in prune:
95 prune[region] = []
96 prune[region].append(ami_id)
97 else:
98 if build_name not in after:
99 after[build_name] = {}
100 if release not in after[build_name]:
101 after[build_name][release] = {}
102 after[build_name][release][ami_name] = info
103
104# scan all regions for AMIs
105AWS = boto3.session.Session()
106for region in AWS.get_available_regions('ec2'):
107 print("* scanning: " + region + '...')
108 EC2 = AWS.client('ec2', region_name=region)
109
110 for image in EC2.describe_images(Owners=['self'])['Images']:
111
112 action = '? UNKNOWN'
113 if region in prune and image['ImageId'] in prune[region]:
114 action = '- REMOVING'
115 elif region in known and image['ImageId'] in known[region]:
116 action = '+ KEEPING'
117
118 print(' ' + action + ': ' + image['Name'] +
119 "\n = " + image['ImageId'], end='', flush=True)
120 if action[0] == '-':
121 EC2.deregister_image(ImageId=image['ImageId'])
122 for blockdev in image['BlockDeviceMappings']:
123 if 'Ebs' in blockdev:
124 print(', ' + blockdev['Ebs']['SnapshotId'],
125 end='', flush=True)
126 if action[0] == '-':
127 EC2.delete_snapshot(
128 SnapshotId=blockdev['Ebs']['SnapshotId'])
129 print()
130
131# update releases/<profile>.yaml
132with open(RELEASE_YAML, 'w') as data:
133 yaml.dump(after, data, sort_keys=False)
diff --git a/scripts/resolve-profile.py.in b/scripts/resolve-profile.py.in
new file mode 100644
index 0000000..d784cf3
--- /dev/null
+++ b/scripts/resolve-profile.py.in
@@ -0,0 +1,105 @@
1@PYTHON@
2# vim: set ts=4 et:
3
4import json
5import os
6import shutil
7import sys
8from datetime import datetime, timedelta
9from pyhocon import ConfigFactory
10
11if len(sys.argv) != 2:
12 sys.exit("Usage: " + os.path.basename(__file__) + " <profile>")
13
14PROFILE = sys.argv[1]
15
16SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
17
18# path to the profile config file
19PROFILE_CONF = os.path.join(SCRIPT_DIR, '..', 'profiles', PROFILE + '.conf')
20
21# load the profile's build configuration
22BUILDS = ConfigFactory.parse_file(PROFILE_CONF)['BUILDS']
23
24# where we store the profile's builds' config/output
25PROFILE_DIR = os.path.join(SCRIPT_DIR, 'profile', PROFILE)
26if not os.path.exists(PROFILE_DIR):
27 os.makedirs(PROFILE_DIR)
28
29# fold these build config keys' dict to scalar
30FOLD_DICTS = {
31 'ami_access': ',{0}',
32 'ami_regions': ',{0}',
33 'repos': "\n@{1} {0}",
34 'pkgs': ' {0}@{1}',
35 'kernel_modules': ',{0}',
36 'kernel_options': ' {0}'
37}
38
39NOW = datetime.utcnow()
40ONE_DAY = timedelta(days=1)
41
42
43# func to fold dict down to scalar
44def fold(fdict, ffmt):
45 folded = ''
46 for fkey, fval in fdict.items():
47 fkey = fkey.strip('"') # complex keys may be in quotes
48 if fval is True:
49 folded += ffmt[0] + fkey
50 elif not (fval is None or fval is False):
51 folded += ffmt.format(fkey, fval)
52 return folded[1:]
53
54
55# parse/resolve HOCON profile's builds' config
56for build, cfg in BUILDS.items():
57 build_dir = os.path.join(PROFILE_DIR, build)
58
59 # make a fresh profile build directory
60 if os.path.exists(build_dir):
61 shutil.rmtree(build_dir)
62 os.makedirs(build_dir)
63
64 # populate profile build vars
65 cfg['profile'] = PROFILE
66 cfg['profile_build'] = build
67
68 # mostly edge-related temporal substitutions
69 if cfg['end_of_life'] == '@TOMORROW@':
70 cfg['end_of_life'] = (NOW + ONE_DAY).isoformat(timespec='seconds')
71 elif cfg['end_of_life'] is not None:
72 # to explicitly UTC-ify end_of_life
73 cfg['end_of_life'] = datetime.fromisoformat(
74 cfg['end_of_life'] + '+00:00').isoformat(timespec='seconds')
75 if cfg['revision'] == '@NOW@':
76 cfg['revision'] = NOW.strftime('%Y%m%d%H%M%S')
77
78 # fold dict vars to scalars
79 for foldkey, foldfmt in FOLD_DICTS.items():
80 cfg[foldkey] = fold(cfg[foldkey], foldfmt)
81
82 # fold 'svcs' dict to scalar
83 lvls = {}
84 for svc, lvl in cfg['svcs'].items():
85 if lvl is True:
86 # service in default runlevel
87 lvls['default'].append(svc)
88 elif not (lvl is None or lvl is False):
89 # service in specified runlevel (skip svc when false/null)
90 if lvl not in lvls.keys():
91 lvls[lvl] = []
92 lvls[lvl].append(svc)
93 cfg['svcs'] = ' '.join(
94 str(lvl) + '=' + ','.join(
95 str(svc) for svc in svcs
96 ) for lvl, svcs in lvls.items()
97 )
98
99 # resolve ami_name and ami_desc
100 cfg['ami_name'] = cfg['ami_name'].format(var=cfg)
101 cfg['ami_desc'] = cfg['ami_desc'].format(var=cfg)
102
103 # write build vars file
104 with open(os.path.join(build_dir, 'vars.json'), 'w') as out:
105 json.dump(cfg, out, indent=4, separators=(',', ': '))
diff --git a/scripts/setup-ami b/scripts/setup-ami
new file mode 100755
index 0000000..aaa0472
--- /dev/null
+++ b/scripts/setup-ami
@@ -0,0 +1,344 @@
1#!/bin/sh
2# vim: set ts=4 et:
3
4set -eu
5
6DEVICE=/dev/xvdf
7TARGET=/mnt/target
8
9# what bootloader should we use?
10[ -d "/sys/firmware/efi" ] && BOOTLOADER=grub-efi || BOOTLOADER=syslinux
11
12die() {
13 printf '\033[1;31mERROR:\033[0m %s\n' "$@" >&2 # bold red
14 exit 1
15}
16
17einfo() {
18 printf '\n\033[1;36m> %s\033[0m\n' "$@" >&2 # bold cyan
19}
20
21rc_add() {
22 runlevel="$1"; shift # runlevel name
23 services="$*" # names of services
24
25 for svc in $services; do
26 mkdir -p "$TARGET/etc/runlevels/$runlevel"
27 ln -s "/etc/init.d/$svc" "$TARGET/etc/runlevels/$runlevel/$svc"
28 echo " * service $svc added to runlevel $runlevel"
29 done
30}
31
32wgets() (
33 url="$1" # url to fetch
34 sha256="$2" # expected SHA256 sum of output
35 dest="$3" # output path and filename
36
37 wget -T 10 -q -O "$dest" "$url"
38 echo "$sha256 $dest" | sha256sum -c > /dev/null
39)
40
41validate_block_device() {
42 lsblk -P --fs "$DEVICE" >/dev/null 2>&1 || \
43 die "'$DEVICE' is not a valid block device"
44
45 if lsblk -P --fs "$DEVICE" | grep -vq 'FSTYPE=""'; then
46 die "Block device '$DEVICE' is not blank"
47 fi
48}
49
50fetch_apk_tools() {
51 store="$(mktemp -d)"
52 tarball="$(basename "$APK_TOOLS")"
53
54 wgets "$APK_TOOLS" "$APK_TOOLS_SHA256" "$store/$tarball"
55 tar -C "$store" -xf "$store/$tarball"
56
57 find "$store" -name apk
58}
59
60# mostly from Alpine's /sbin/setup-disk
61setup_partitions() {
62 start=1M # TODO: do we really need to waste 1M?
63 line=
64
65 # create new partitions
66 (
67 for line in "$@"; do
68 case "$line" in
69 0M*) ;;
70 *) echo "$start,$line"; start= ;;
71 esac
72 done
73 ) | sfdisk --quiet --label dos "$DEVICE"
74
75 # we assume that the build host will create the new devices within 5s
76 tries=5
77 while [ ! -e "${DEVICE}1" ]; do
78 [ $tries -eq 0 ] && break
79 sleep 1
80 tries=$(( tries - 1 ))
81 done
82 [ -e "${DEVICE}1" ] || die "Expected new device ${DEVICE}1 not created"
83}
84
85make_filesystem() {
86 root_dev="$DEVICE"
87
88 if [ "$BOOTLOADER" = 'grub-efi' ]; then
89 # create a small EFI partition (remainder for root), and mount it
90 setup_partitions '5M,EF' ',L'
91 root_dev="${DEVICE}2"
92 mkfs.vfat -n EFI "${DEVICE}1"
93 fi
94
95 mkfs.ext4 -O ^64bit -L / "$root_dev"
96 mount "$root_dev" "$TARGET"
97
98 if [ "$BOOTLOADER" = 'grub-efi' ]; then
99 mkdir -p "$TARGET/boot/efi"
100 mount -t vfat "${DEVICE}1" "$TARGET/boot/efi"
101 fi
102}
103
104setup_repositories() {
105 mkdir -p "$TARGET/etc/apk/keys"
106 echo "$REPOS" > "$TARGET/etc/apk/repositories"
107}
108
109fetch_keys() {
110 tmp="$(mktemp -d)"
111
112 wgets "$ALPINE_KEYS" "$ALPINE_KEYS_SHA256" "$tmp/alpine-keys.apk"
113 tar -C "$TARGET" --warning=no-unknown-keyword -xvf "$tmp/alpine-keys.apk" etc/apk/keys
114 rm -rf "$tmp"
115}
116
117install_base() {
118 $apk add --root "$TARGET" --no-cache --initdb alpine-base
119 # verify release matches
120 if [ "$VERSION" != "edge" ]; then
121 ALPINE_RELEASE=$(cat "$TARGET/etc/alpine-release")
122 [ "$RELEASE" = "$ALPINE_RELEASE" ] || \
123 die "Newer Alpine release detected: $ALPINE_RELEASE"
124 fi
125}
126
127setup_chroot() {
128 mount -t proc none "$TARGET/proc"
129 mount --bind /dev "$TARGET/dev"
130 mount --bind /sys "$TARGET/sys"
131
132 # Needed for bootstrap, will be removed in the cleanup stage.
133 install -Dm644 /etc/resolv.conf "$TARGET/etc/resolv.conf"
134}
135
136install_core_packages() {
137 chroot "$TARGET" apk --no-cache add $PKGS
138 chroot "$TARGET" apk --no-cache add --no-scripts $BOOTLOADER
139
140 # Disable starting getty for physical ttys because they're all inaccessible
141 # anyhow. With this configuration boot messages will still display in the
142 # EC2 console.
143 sed -Ei '/^tty[0-9]/s/^/#/' "$TARGET/etc/inittab"
144
145 # Make it a little more obvious who is logged in by adding username to the
146 # prompt
147 sed -i "s/^export PS1='/&\\\\u@/" "$TARGET/etc/profile"
148}
149
150setup_mdev() {
151 cp /tmp/nvme-ebs-links "$TARGET/lib/mdev"
152 # insert nvme ebs mdev configs just above "# fallback" comment
153 sed -n -i -e '/# fallback/r /tmp/nvme-ebs-mdev.conf' -e 1x -e '2,${x;p}' -e '${x;p}' "$TARGET/etc/mdev.conf"
154}
155
156create_initfs() {
157 # Enable ENA and NVME features these don't hurt for any instance and are
158 # hard requirements of the 5 series and i3 series of instances
159 # TODO: profile-ize?
160 sed -Ei 's/^features="([^"]+)"/features="\1 nvme ena"/' \
161 "$TARGET/etc/mkinitfs/mkinitfs.conf"
162
163 chroot "$TARGET" /sbin/mkinitfs $(basename $(find "$TARGET/lib/modules/"* -maxdepth 0))
164}
165
166install_bootloader() {
167 case "$BOOTLOADER" in
168 syslinux) install_extlinux ;;
169 grub-efi) install_grub_efi ;;
170 *) die "unknown bootloader '$BOOTLOADER'" ;;
171 esac
172}
173
174install_extlinux() {
175 # Must use disk labels instead of UUID or devices paths so that this works
176 # across instance familes. UUID works for many instances but breaks on the
177 # NVME ones because EBS volumes are hidden behind NVME devices.
178 #
179 # Enable ext4 because the root device is formatted ext4
180 #
181 # Shorten timeout (1/10s) as EC2 has no way to interact with instance console
182 #
183 # ttyS0 is the target for EC2s "Get System Log" feature whereas tty0 is the
184 # target for EC2s "Get Instance Screenshot" feature. Enabling the serial
185 # port early in extlinux gives the most complete output in the system log.
186 sed -Ei -e "s|^[# ]*(root)=.*|\1=LABEL=/|" \
187 -e "s|^[# ]*(default_kernel_opts)=.*|\1=\"$KERNEL_OPTS\"|" \
188 -e "s|^[# ]*(serial_port)=.*|\1=ttyS0|" \
189 -e "s|^[# ]*(modules)=.*|\1=$KERNEL_MODS|" \
190 -e "s|^[# ]*(default)=.*|\1=virt|" \
191 -e "s|^[# ]*(timeout)=.*|\1=1|" \
192 "$TARGET/etc/update-extlinux.conf"
193
194 chroot "$TARGET" /sbin/extlinux --install /boot
195 chroot "$TARGET" /sbin/update-extlinux --warn-only
196}
197
198# TODO: this isn't quite working for some reason
199install_grub_efi() {
200 case "$ARCH" in
201 x86_64) grub_target=x86_64-efi ; fwa=x64 ;;
202 aarch64) grub_target=arm64-efi ; fwa=aa64 ;;
203 *) die "ARCH=$ARCH is currently unsupported" ;;
204 esac
205
206 # disable nvram so grub doesn't call efibootmgr
207 chroot "$TARGET" /usr/sbin/grub-install --target="$grub_target" --efi-directory=/boot/efi \
208 --bootloader-id=alpine --boot-directory=/boot --no-nvram
209
210 # fallback mode
211 install -D "$TARGET/boot/efi/EFI/alpine/grub$fwa.efi" "$TARGET/boot/efi/EFI/boot/$fwa.efi"
212
213 # add cmdline linux defaults to /etc/default/grub
214 echo "GRUB_CMDLINE_LINUX_DEFAULT=\"modules=$KERNEL_MODS $KERNEL_OPTS\"" >> "$TARGET"/etc/default/grub
215
216 # eliminate grub pause
217 sed -ie 's/^GRUB_TIMEOUT=.$/GRUB_TIMEOUT=0/' "$TARGET/etc/default/grub"
218
219 # generate/install new config
220 [ -e "$TARGET/boot/grub/grub.cfg" ] && cp "$TARGET/boot/grub/grub.cfg" "$TARGET/boot/grub/grub.cfg.backup"
221 chroot "$TARGET" grub-mkconfig -o /boot/grub/grub.cfg
222}
223
224setup_fstab() {
225 cat > "$TARGET/etc/fstab" <<EOF
226# <fs> <mountpoint> <type> <opts> <dump/pass>
227LABEL=/ / ext4 defaults,noatime 1 1
228EOF
229
230 # if we're using grub-efi bootloader, add extra line for EFI partition
231 if [ "$BOOTLOADER" = 'grub-efi' ]; then
232 echo "LABEL=EFI /boot/efi vfat defaults,noatime,uid=0,gid=0,umask=077 0 0" >> "$TARGET/etc/fstab"
233 fi
234}
235
236setup_networking() {
237 cat > "$TARGET/etc/network/interfaces" <<EOF
238auto lo
239iface lo inet loopback
240
241auto eth0
242iface eth0 inet dhcp
243EOF
244}
245
246enable_services() {
247 for lvl_svcs in $SVCS; do
248 rc_add $(echo "$lvl_svcs" | tr '=,' ' ')
249 done
250}
251
252# TODO: allow profile to specify alternate ALPINE_USER?
253# NOTE: tiny-ec2-bootstrap will need to be updated to support that!
254create_alpine_user() {
255 # Allow members of the wheel group to sudo without a password. By default
256 # this will only be the alpine user. This allows us to ship an AMI that is
257 # accessible via SSH using the user's configured SSH keys (thanks to
258 # tiny-ec2-bootstrap) but does not allow remote root access which is the
259 # best-practice.
260 sed -i '/%wheel .* NOPASSWD: .*/s/^# //' "$TARGET/etc/sudoers"
261
262 # There is no real standard ec2 username across AMIs, Amazon uses ec2-user
263 # for their Amazon Linux AMIs but Ubuntu uses ubuntu, Fedora uses fedora,
264 # etc... (see: https://alestic.com/2014/01/ec2-ssh-username/). So our user
265 # and group are alpine because this is Alpine Linux. On instance bootstrap
266 # the user can create whatever users they want and delete this one.
267 chroot "$TARGET" /usr/sbin/addgroup alpine
268 chroot "$TARGET" /usr/sbin/adduser -h /home/alpine -s /bin/sh -G alpine -D alpine
269 chroot "$TARGET" /usr/sbin/addgroup alpine wheel
270 chroot "$TARGET" /usr/bin/passwd -u alpine
271}
272
273configure_ntp() {
274 # EC2 provides an instance-local NTP service syncronized with GPS and
275 # atomic clocks in-region. Prefer this over external NTP hosts when running
276 # in EC2.
277 #
278 # See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html
279 sed -e 's/^pool /server /' \
280 -e 's/pool.ntp.org/169.254.169.123/g' \
281 -i "$TARGET/etc/chrony/chrony.conf"
282}
283
284cleanup() {
285 # Sweep cruft out of the image that doesn't need to ship or will be
286 # re-generated when the image boots
287 rm -f \
288 "$TARGET/var/cache/apk/"* \
289 "$TARGET/etc/resolv.conf" \
290 "$TARGET/root/.ash_history" \
291 "$TARGET/etc/"*-
292
293 [ "$BOOTLOADER" = 'grub-efi' ] && umount "$TARGET/boot/efi"
294
295 umount \
296 "$TARGET/dev" \
297 "$TARGET/proc" \
298 "$TARGET/sys"
299
300 umount "$TARGET"
301}
302
303main() {
304 validate_block_device
305
306 [ -d "$TARGET" ] || mkdir "$TARGET"
307
308 einfo "Fetching static APK tools"
309 apk="$(fetch_apk_tools)"
310
311 einfo "Creating root filesystem"
312 make_filesystem
313
314 einfo "Configuring Alpine repositories"
315 setup_repositories
316
317 einfo "Fetching Alpine signing keys"
318 fetch_keys
319
320 einfo "Installing base system"
321 install_base
322
323 setup_chroot
324
325 einfo "Installing core packages"
326 install_core_packages
327
328 einfo "Configuring and enabling boot loader"
329 create_initfs
330 install_bootloader
331
332 einfo "Configuring system"
333 setup_mdev
334 setup_fstab
335 setup_networking
336 enable_services
337 create_alpine_user
338 configure_ntp
339
340 einfo "All done, cleaning up"
341 cleanup
342}
343
344main "$@"
diff --git a/scripts/update-release.py.in b/scripts/update-release.py.in
new file mode 100644
index 0000000..95350c9
--- /dev/null
+++ b/scripts/update-release.py.in
@@ -0,0 +1,62 @@
1@PYTHON@
2# vim: set ts=4 et:
3
4import json
5import os
6import re
7import sys
8import yaml
9
10if len(sys.argv) != 3:
11 sys.exit("Usage: " + os.path.basename(__file__) + " <profile> <build>")
12
13PROFILE = sys.argv[1]
14BUILD = sys.argv[2]
15
16SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
17MANIFEST_JSON = os.path.join(
18 SCRIPT_DIR, 'profile', PROFILE, BUILD, 'manifest.json'
19)
20
21RELEASE_DIR = os.path.join(SCRIPT_DIR, '..', 'releases')
22RELEASE_YAML = os.path.join(RELEASE_DIR, PROFILE + '.yaml')
23
24if not os.path.exists(RELEASE_DIR):
25 os.makedirs(RELEASE_DIR)
26
27releases = {}
28if os.path.exists(RELEASE_YAML):
29 with open(RELEASE_YAML, 'r') as data:
30 releases = yaml.safe_load(data)
31
32with open(MANIFEST_JSON, 'r') as data:
33 MANIFEST = json.load(data)
34
35A = re.split(':|,', MANIFEST['builds'][0]['artifact_id'])
36ARTIFACTS = dict(zip(A[0::2], A[1::2]))
37BUILD_TIME = MANIFEST['builds'][0]['build_time']
38DATA = MANIFEST['builds'][0]['custom_data']
39RELEASE = DATA['release']
40
41if BUILD not in releases:
42 releases[BUILD] = {}
43if RELEASE not in releases[BUILD]:
44 releases[BUILD][RELEASE] = {}
45
46REVISION = {
47 'description': DATA['ami_desc'],
48 'profile': PROFILE,
49 'profile_build': BUILD,
50 'version': DATA['version'],
51 'release': RELEASE,
52 'arch': DATA['arch'],
53 'revision': DATA['revision'],
54 'end_of_life': DATA['end_of_life'],
55 'build_time': BUILD_TIME,
56 'artifacts': ARTIFACTS
57}
58
59releases[BUILD][RELEASE][DATA['ami_name']] = REVISION
60
61with open(RELEASE_YAML, 'w') as data:
62 yaml.dump(releases, data, sort_keys=False)
diff --git a/scrub-old-amis.py.in b/scrub-old-amis.py.in
deleted file mode 100644
index 34d7be3..0000000
--- a/scrub-old-amis.py.in
+++ /dev/null
@@ -1,72 +0,0 @@
1@PYTHON@
2
3import re
4import yaml
5import boto3
6
7
8# All Alpine AMIs should match this regex if they're valid
9AMI_RE = re.compile("^Alpine-(\d+\.\d+)(?:-r(\d+))?-Hardened-EC2")
10
11
12# Load current AMI version from config
13with open("alpine-ami.yaml") as fp:
14 ami_cfg = yaml.full_load(fp)["variables"]
15 current = (float(ami_cfg["alpine_release"]), int(ami_cfg["ami_release"]))
16
17
18# Fetch all matching AMIs
19amis = []
20
21for region in boto3.session.Session().get_available_regions("ec2"):
22 ec2 = boto3.client("ec2", region_name=region)
23
24 for image in ec2.describe_images(Owners=["self"])["Images"]:
25 match = AMI_RE.match(image["Name"])
26 if not match:
27 continue
28
29 os_rel, ami_rel = match.groups()
30 amis.append((
31 region, image["ImageId"],
32 image["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"],
33 float(os_rel), int(ami_rel) if ami_rel else 0))
34
35
36# Determine the set to discard based region and version
37ok_regions = set()
38discards = []
39
40# Cluster candidates by region/version pair, newest in a region first.
41# This should result in the first match for a region always being the newest
42# AMI for that region and all subsequent matches in the region being old.
43# Even so we must keep track of regions with current images on the off-chance
44# that a region only has old images. In that case we want to preserve the old
45# images till we can publish new ones manually so users can still launch
46# Alpine systems without interruption.
47candidates = sorted(amis, key=lambda i: (i[0], (i[1], i[3])), reverse=True)
48
49for ami in candidates:
50 (region, ami, snapshot), version = ami[:3], ami[3:]
51
52 if version > current:
53 print("{} has AMI '{}' newer than current".format(region, ami))
54 continue
55 elif version == current:
56 ok_regions.add(region)
57 continue
58 elif version < current and region in ok_regions:
59 discards.append((region, ami, snapshot))
60 else:
61 print("Not discarding old image in {}".format(region))
62 continue
63
64
65# Scrub the old ones
66for region, image, snapshot in discards:
67 print("Removing image '{}', snapshot '{}' in {}".format(
68 image, snapshot, region))
69
70 ec2 = boto3.client("ec2", region_name=region)
71 ec2.deregister_image(ImageId=image)
72 ec2.delete_snapshot(SnapshotId=snapshot)
diff --git a/variables.yaml-default b/variables.yaml-default
deleted file mode 100644
index 0cd3cc2..0000000
--- a/variables.yaml-default
+++ /dev/null
@@ -1,77 +0,0 @@
1### Builder-Instance Options ###
2
3# Region to build in, if we initiate a build from outside AWS
4region:
5
6# Subnet ID in which the builder instance is to be launched. VPC will be
7# automatically determined.
8subnet:
9
10# Optional security group to apply to the builder instance
11security_group:
12
13# By default, public IPs are assigned (or not) per the subnet's configuration.
14# Set to "true" or "false" to explicitly override the subnet's public IP auto-
15# assign configuration.
16public_ip: ""
17
18
19### Build Options ###
20
21# Uncomment/increment every for every rebuild of an Alpine release;
22# re-comment/zero for every new Alpine release
23#revision: "-1"
24
25# AMI name prefix and suffix
26ami_name_prefix: "alpine-ami-"
27ami_name_suffix: ""
28
29# AMI description prefix and suffix
30ami_desc_prefix: "Alpine Linux "
31ami_desc_suffix: " - https://github.com/mcrute/alpine-ec2-ami"
32
33# List of custom lines to add to /etc/apk/repositories
34add_repos:
35# - "@my-repo http://my-repo.tld/path"
36
37# List of additional packages to add to the AMI.
38add_pkgs:
39# - package-name
40
41# Additional services to start at the specified level
42add_svcs:
43# boot:
44# - service1
45# default:
46# - service2
47
48# Size of the AMI image (in GiB).
49volume_size: "1"
50
51# Encrypt the AMI?
52encrypt_ami: "false"
53
54# List of groups that should have access to the AMI. However, only two
55# values are currently supported: 'all' for public, '' or unset for private.
56ami_access:
57 - "all"
58
59# List of regions to where the AMI should be copied
60deploy_regions:
61 - "us-east-1"
62 - "us-east-2"
63 - "us-west-1"
64 - "us-west-2"
65 - "ca-central-1"
66 - "eu-central-1"
67 - "eu-north-1"
68 - "eu-west-1"
69 - "eu-west-2"
70 - "eu-west-3"
71 - "ap-northeast-1"
72 - "ap-northeast-2"
73# - "ap-northeast-3" # skipped, available by subscription only
74 - "ap-southeast-1"
75 - "ap-southeast-2"
76 - "ap-south-1"
77 - "sa-east-1"
© 2015 Mike Crute