aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJake Buchholz <tomalok@gmail.com>2019-04-17 16:39:47 -0700
committerJake Buchholz <tomalok@gmail.com>2019-04-17 16:39:47 -0700
commit0d63605eabc3dc1016f1693589845f478657103c (patch)
tree1eb00f5c680aeec75ff973d754ff9e140b456755
parent24144391d61723da5217539f7b22b5ea37b959f0 (diff)
downloadalpine-ec2-ami-0d63605eabc3dc1016f1693589845f478657103c.tar.bz2
alpine-ec2-ami-0d63605eabc3dc1016f1693589845f478657103c.tar.xz
alpine-ec2-ami-0d63605eabc3dc1016f1693589845f478657103c.zip
Introducing Build Profiles
This is a big update, directly related to issue #37 (build profiles), reorganizes 'variables.yaml' into base, version, and arch core profiles, which are merged (and optionally tweaked) in build profiles. Foundations are also laid to progress issues #28 (aarch64 AMIs), #22 (Setup CI), #20 (update generate/cleanup scripts), and #23 (retention policy). Although there are still a couple of improvements I'm working on with regards to setup-ami.sh improvements, building AMIs from profiles is working... make PROFILE=test -- all builds of the test profile make PROFILE=test BUILD=test-edge-x86_64 -- just the edge build of the test profile ...and I'm hoping to get some feedback with what's been done so far.
Diffstat
-rw-r--r--Makefile66
-rw-r--r--alpine-ami.yaml70
-rw-r--r--packer.yaml73
-rw-r--r--profiles/alpine-amis.yaml48
-rw-r--r--profiles/arch/aarch64-3.9.yaml13
-rw-r--r--profiles/arch/aarch64-current.yaml12
-rw-r--r--profiles/arch/aarch64-edge.yaml18
-rw-r--r--profiles/arch/x86_64-3.9.yaml13
-rw-r--r--profiles/arch/x86_64-current.yaml13
-rw-r--r--profiles/arch/x86_64-edge.yaml18
-rw-r--r--profiles/base/1.yaml78
-rw-r--r--profiles/base/current.yaml13
-rw-r--r--profiles/base/edge.yaml18
-rw-r--r--profiles/test.yaml48
-rw-r--r--profiles/version/3.9.yaml15
-rw-r--r--profiles/version/current.yaml13
-rw-r--r--profiles/version/edge.yaml23
-rwxr-xr-xscripts/make-amis13
-rwxr-xr-xscripts/nvme/nvme-ebs-links (renamed from nvme/nvme-ebs-links)0
-rw-r--r--scripts/nvme/nvme-ebs-mdev.conf (renamed from nvme/nvme-ebs-mdev.conf)0
-rw-r--r--scripts/resolve-profile.py.in82
-rwxr-xr-xscripts/setup-ami.sh (renamed from make_ami.sh)83
-rw-r--r--scripts/yaml2json.py.in9
-rw-r--r--variables.yaml-default77
24 files changed, 566 insertions, 250 deletions
diff --git a/Makefile b/Makefile
index a624e9e..acfb9f2 100644
--- a/Makefile
+++ b/Makefile
@@ -1,44 +1,28 @@
1.PHONY: ami 1ALL_SCRIPTS := $(wildcard scripts/*)
2 2ALL_PROFILES := $(shell find profiles -name '*.yaml' -type f)
3ami: convert 3CORE_PROFILES := $(shell for i in base version arch; do ls profiles/$$i/*.yaml | sort -V; done)
4 packer build -var-file=build/variables.json build/alpine-ami.json 4PROFILE := default
5 5BUILD :=
6edge: convert 6
7 @echo '{ "version": "edge", "release": "edge", "revision": "'-`date +%Y%m%d%H%M%S`'" }' > build/edge.json 7.PHONY: amis
8 packer build -var-file=build/variables.json -var-file=build/edge.json build/alpine-ami.json 8amis: build build/profile build/packer.json
9 9 build/make-amis $(BUILD)
10convert: build/convert 10
11 [ -f variables.yaml ] || cp variables.yaml-default variables.yaml 11build: $(SCRIPTS)
12 build/convert variables.yaml > build/variables.json 12 [ -d build ] || mkdir build
13 build/convert alpine-ami.yaml > build/alpine-ami.json 13 python3 -m venv build/.py3
14 14 build/.py3/bin/pip install pyyaml boto3
15build/convert: 15 (cd build; for i in $(ALL_SCRIPTS); do ln -sf ../$$i .; done)
16 [ -d ".py3" ] || python3 -m venv .py3 16
17 .py3/bin/pip install pyyaml boto3 17build/profile: build build/resolve-profile.py $(ALL_PROFILES)
18 18 cat $(CORE_PROFILES) profiles/$(PROFILE).yaml | build/resolve-profile.py $(PROFILE)
19 [ -d "build" ] || mkdir build 19
20 20build/packer.json: build build/yaml2json.py packer.yaml
21 # Make stupid simple little YAML/JSON converter so we can maintain our 21 build/yaml2json.py packer.yaml > build/packer.json
22 # packer configs in a sane format that allows comments but also use packer 22
23 # which only supports JSON 23%.py: %.py.in build
24 @echo "#!`pwd`/.py3/bin/python" > build/convert 24 sed "s|@PYTHON@|#!`pwd`/build/.py3/bin/python|" $< > $@
25 @echo "import yaml, json, sys" >> build/convert
26 @echo "y = yaml.full_load(open(sys.argv[1]))" >> build/convert
27 @echo "for k in ['ami_access','deploy_regions','add_repos','add_pkgs','add_svcs']:" >> build/convert
28 @echo " if k in y and isinstance(y[k], list):" >> build/convert
29 @echo " y[k] = ','.join(str(x) for x in y[k])" >> build/convert
30 @echo " if k in y and isinstance(y[k], dict):" >> build/convert
31 @echo " y[k] = ':'.join(str(l) + '=' + ','.join(str(s) for s in ss) for l, ss in y[k].items())" >> build/convert
32 @echo "json.dump(y, sys.stdout, indent=4, separators=(',', ': '))" >> build/convert
33 @chmod +x build/convert
34
35%.py: %.py.in
36 sed "s|@PYTHON@|#!`pwd`/.py3/bin/python|" $< > $@
37 chmod +x $@ 25 chmod +x $@
38 26
39.PHONY: clean
40clean: 27clean:
41 rm -rf build .py3 scrub-old-amis.py gen-readme.py 28 rm -rf build scrub-old-amis.py gen-readme.py
42
43distclean: clean
44 rm -f variables.yaml
diff --git a/alpine-ami.yaml b/alpine-ami.yaml
deleted file mode 100644
index 1505055..0000000
--- a/alpine-ami.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
1variables:
2
3 # NOTE: Configuration is done with a 'variables.yaml' file. If it doesn't
4 # exist, default configuration is copied from 'variables.yaml-default'.
5
6 # NOTE: Changing arch/version/release may require modifying 'make_ami.sh'.
7 arch: x86_64
8 version: "3.9"
9 release: "3.9.3"
10 revision: ""
11
12builders:
13 - type: "amazon-ebssurrogate"
14
15 ### Builder Instance Details
16
17 region: "{{user `region`}}"
18 subnet_id: "{{user `subnet`}}"
19 security_group_id: "{{user `security_group`}}"
20 instance_type: "t3.nano"
21 associate_public_ip_address: "{{user `public_ip`}}"
22 launch_block_device_mappings:
23 - volume_type: "gp2"
24 device_name: "/dev/xvdf"
25 delete_on_termination: "true"
26 volume_size: "{{user `volume_size`}}"
27 ssh_username: "ec2-user"
28 source_ami_filter:
29 # use the latest Amazon Linux AMI
30 filters:
31 virtualization-type: "hvm"
32 root-device-type: "ebs"
33 architecture: "x86_64"
34 name: "amzn2-ami-hvm-2.0.*-gp2"
35 owners:
36 - "137112412989"
37 most_recent: "true"
38
39 ### AMI Build Details
40
41 ami_name: "{{user `ami_name_prefix`}}{{user `release`}}{{user `revision`}}-{{user `arch`}}{{user `ami_name_suffix`}}"
42 ami_description: "{{user `ami_desc_prefix`}}{{user `release`}}{{user `revision`}} {{user `arch`}}{{user `ami_desc_suffix`}}"
43 ami_virtualization_type: "hvm"
44 ami_root_device:
45 source_device_name: "/dev/xvdf"
46 device_name: "/dev/xvda"
47 delete_on_termination: "true"
48 volume_size: "{{user `volume_size`}}"
49 volume_type: "gp2"
50 encrypt_boot: "{{user `encrypt_ami`}}"
51 ena_support: "true"
52 sriov_support: "true"
53 ami_groups: "{{user `ami_access`}}"
54 ami_regions: "{{user `deploy_regions`}}"
55
56
57provisioners:
58 - type: "file"
59 source: "nvme/"
60 destination: "/tmp"
61 - type: "shell"
62 script: "make_ami.sh"
63 environment_vars:
64 - "VERSION={{user `version`}}"
65 - "RELEASE={{user `release`}}"
66 - "REVISION={{user `revision`}}"
67 - "ADD_REPOS='{{user `add_repos`}}'"
68 - "ADD_PKGS='{{user `add_pkgs`}}'"
69 - "ADD_SVCS='{{user `add_svcs`}}'"
70 execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }}"'
diff --git a/packer.yaml b/packer.yaml
new file mode 100644
index 0000000..75ac5b2
--- /dev/null
+++ b/packer.yaml
@@ -0,0 +1,73 @@
1builders:
2 - type: amazon-ebssurrogate
3
4 ### Builder Instance Details
5
6 region: '{{user `build_region`}}'
7 subnet_id: '{{user `build_subnet`}}'
8 security_group_id: '{{user `build_security_group`}}'
9 instance_type: '{{user `build_instance_type`}}'
10 associate_public_ip_address: '{{user `build_public_ip`}}'
11 source_ami_filter:
12 # use the latest Amazon Linux AMI
13 owners:
14 - '{{user `build_ami_owner`}}'
15 most_recent: '{{user `build_ami_latest`}}'
16 filters:
17 virtualization-type: hvm
18 root-device-type: ebs
19 architecture: '{{user `build_arch`}}'
20 name: '{{user `build_ami_name`}}'
21 launch_block_device_mappings:
22 - volume_type: gp2
23 device_name: /dev/xvdf
24 delete_on_termination: 'true'
25 volume_size: '{{user `ami_volume_size`}}'
26 ssh_username: '{{user `build_user`}}'
27
28 ### AMI Build Details
29
30 ami_name: '{{user `ami_name_prefix`}}{{user `release`}}-{{user `revision`}}-{{user `arch`}}{{user `ami_name_suffix`}}'
31 ami_description: '{{user `ami_desc_prefix`}}{{user `release`}}-{{user `revision`}} {{user `arch`}}{{user `ami_desc_suffix`}}'
32 ami_virtualization_type: hvm
33 ami_root_device:
34 volume_type: gp2
35 source_device_name: /dev/xvdf
36 device_name: /dev/xvda
37 delete_on_termination: 'true'
38 volume_size: '{{user `ami_volume_size`}}'
39 encrypt_boot: '{{user `ami_encrypt`}}'
40 ena_support: 'true'
41 sriov_support: 'true'
42 ami_groups: '{{user `ami_access`}}'
43 ami_regions: '{{user `ami_regions`}}'
44
45
46provisioners:
47 - type: file
48 source: nvme/
49 destination: /tmp
50 - type: shell
51 script: setup-ami.sh
52 environment_vars:
53 - "VERSION='{{user `version`}}'"
54 - "RELEASE='{{user `release`}}'"
55 - "REVISION='{{user `revision`}}'"
56 - "ARCH='{{user `arch`}}'"
57 - "REPOS='{{user `repos`}}'"
58 - "PKGS='{{user `pkgs`}}'"
59 - "SVCS='{{user `svcs`}}'"
60 execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }}"'
61
62
63post-processors:
64 - type: manifest
65 output: 'profile/{{user `profile_build`}}/manifest.json'
66 custom_data:
67 profile: '{{user `profile`}}'
68 build: '{{user `profile_build`}}'
69 arch: '{{user `arch`}}'
70 version: '{{user `version`}}'
71 release: '{{user `release`}}'
72 revision: '{{user `revision`}}'
73 end_of_life: '{{user `end_of_life`}}'
diff --git a/profiles/alpine-amis.yaml b/profiles/alpine-amis.yaml
new file mode 100644
index 0000000..119601f
--- /dev/null
+++ b/profiles/alpine-amis.yaml
@@ -0,0 +1,48 @@
1### Profile for Building the Publically-Available Alpine Linux AMIs
2
3# Profile overrides
4alpine-amis: &alpine-amis
5 revision: '1'
6 build_region: us-west-2
7 build_subnet: subnet-b80c36e2
8 ami_access:
9 - all
10 ami_regions:
11 - ap-northeast-1
12 - ap-northeast-2
13# - ap-northeast-3 # skipped, available by subscription only
14 - ap-southeast-1
15 - ap-southeast-2
16 - ap-south-1
17 - ca-central-1
18 - eu-central-1
19 - eu-north-1
20 - eu-west-1
21 - eu-west-2
22 - eu-west-3
23 - sa-east-1
24 - us-east-1
25 - us-east-2
26 - us-west-1
27 - us-west-2
28
29# Build definitions
30builds:
31 current-x86_64:
32 <<: *_base-current
33 <<: *_version-current
34 <<: *_x86_64-current
35 <<: *alpine-amis
36 repos:
37 <<: *_base-current-repos
38 <<: *_version-current-repos
39 <<: *_x86_64-current-repos
40 pkgs:
41 <<: *_base-current-pkgs
42 <<: *_version-current-pkgs
43 <<: *_x86_64-current-pkgs
44 svcs:
45 <<: *_base-current-svcs
46 <<: *_version-current-svcs
47 <<: *_x86_64-current-svcs
48
diff --git a/profiles/arch/aarch64-3.9.yaml b/profiles/arch/aarch64-3.9.yaml
new file mode 100644
index 0000000..c7ec980
--- /dev/null
+++ b/profiles/arch/aarch64-3.9.yaml
@@ -0,0 +1,13 @@
1### aarch64 Version 3.9 Variables
2
3_aarch64-3.9: &_aarch64-3_9
4 # Architecture
5 arch: aarch64
6 build_arch: arm64
7 build_instance_type: a1.medium
8 bootloader: grub
9
10 # aarch64 repos/pkgs/svcs for 3.9
11 repos: &_aarch64-3_9-repos {}
12 pkgs: &_aarch64-3_9-pkgs {}
13 svcs: &_aarch64-3_9-svcs {}
diff --git a/profiles/arch/aarch64-current.yaml b/profiles/arch/aarch64-current.yaml
new file mode 100644
index 0000000..1b4e624
--- /dev/null
+++ b/profiles/arch/aarch64-current.yaml
@@ -0,0 +1,12 @@
1### aarch64 Current Version Variables
2
3_aarch64-current: &_aarch64-current
4 # currently from 3.9
5 <<: *_aarch64-3_9
6 # re-anchor for current
7 repos: &_aarch64-current-repos
8 <<: *_aarch64-3_9-repos
9 pkgs: &_aarch64-current-pkgs
10 <<: *_aarch64-3_9-pkgs
11 svcs: &_aarch64-current-svcs
12 <<: *_aarch64-3_9-svcs
diff --git a/profiles/arch/aarch64-edge.yaml b/profiles/arch/aarch64-edge.yaml
new file mode 100644
index 0000000..97d76e2
--- /dev/null
+++ b/profiles/arch/aarch64-edge.yaml
@@ -0,0 +1,18 @@
1### aarch64 Edge Variables
2
3_aarch64-edge: &_aarch64-edge
4 # base on current (for now)
5 <<: *_aarch64-current
6 # overrides (if any)
7
8 # aarch64 repos/pkgs/svcs for edge
9 repos: &_aarch64-edge-repos
10 <<: *_aarch64-current-repos
11 # overrides (if any)
12 pkgs: &_aarch64-edge-pkgs
13 <<: *_aarch64-current-pkgs
14 # overrides (if any)
15 svcs: &_aarch64-edge-svcs
16 <<: *_aarch64-current-svcs
17 # overrides (if any)
18
diff --git a/profiles/arch/x86_64-3.9.yaml b/profiles/arch/x86_64-3.9.yaml
new file mode 100644
index 0000000..79b8908
--- /dev/null
+++ b/profiles/arch/x86_64-3.9.yaml
@@ -0,0 +1,13 @@
1### x86_64 Version 3.9 Variables
2
3_x86_64-3.9: &_x86_64-3_9
4 # Architecture
5 arch: x86_64
6 build_arch: x86_64
7 bootloader: syslinux
8
9 # x86_64 repos/pkgs/svcs for 3.9
10 repos: &_x86_64-3_9-repos {}
11 pkgs: &_x86_64-3_9-pkgs {}
12 svcs: &_x86_64-3_9-svcs {}
13
diff --git a/profiles/arch/x86_64-current.yaml b/profiles/arch/x86_64-current.yaml
new file mode 100644
index 0000000..6dc5555
--- /dev/null
+++ b/profiles/arch/x86_64-current.yaml
@@ -0,0 +1,13 @@
1### x86_64 Current Version Variables
2
3_x86_64-current: &_x86_64-current
4 # Current is 3.9
5 <<: *_x86_64-3_9
6 # re-anchor for current
7 repos: &_x86_64-current-repos
8 <<: *_x86_64-3_9-repos
9 pkgs: &_x86_64-current-pkgs
10 <<: *_x86_64-3_9-pkgs
11 svcs: &_x86_64-current-svcs
12 <<: *_x86_64-3_9-svcs
13
diff --git a/profiles/arch/x86_64-edge.yaml b/profiles/arch/x86_64-edge.yaml
new file mode 100644
index 0000000..2e2f1d2
--- /dev/null
+++ b/profiles/arch/x86_64-edge.yaml
@@ -0,0 +1,18 @@
1### x86_64 Edge Variables
2
3_x86_64-edge: &_x86_64-edge
4 # base from current (at the moment)
5 <<: *_x86_64-current
6 # overrides (if any)
7
8 # x86_64 repos/pkgs/svcs for edge
9 repos: &_x86_64-edge-repos
10 <<: *_x86_64-current-repos
11 # overrides (if any)
12 pkgs: &_x86_64-edge-pkgs
13 <<: *_x86_64-current-pkgs
14 # overrides (if any)
15 svcs: &_x86_64-edge-svcs
16 <<: *_x86_64-current-pkgs
17 # overrides (if any)
18
diff --git a/profiles/base/1.yaml b/profiles/base/1.yaml
new file mode 100644
index 0000000..29c7354
--- /dev/null
+++ b/profiles/base/1.yaml
@@ -0,0 +1,78 @@
1### Base Variables, Revsion #1
2
3_base-1: &_base-1
4 # Profile/Build
5 profile:
6 profile_build:
7
8 # Versioning
9 version:
10 release:
11 end_of_life:
12 revision:
13
14 # Architecture
15 arch:
16 build_arch:
17
18 # Builder-instance
19 build_region:
20 build_subnet:
21 build_security_group:
22 build_instance_type: t3.nano
23 build_public_ip: ''
24 build_user: ec2-user
25 build_ami_name: amzn2-ami-hvm-2.0.*-gp2
26 build_ami_owner: '137112412989'
27 build_ami_latest: 'true'
28
29 # AMI build/deploy
30 ami_name_prefix: alpine-ami-
31 ami_name_suffix: ''
32 ami_desc_prefix: 'Alpine Linux '
33 ami_desc_suffix: ' - https://github.com/mcrute/alpine-ec2-ami'
34 ami_volume_size: '1'
35 ami_encrypt: 'false'
36 ami_user: alpine
37 ami_access:
38 ami_regions:
39
40 # AMI configuration
41 bootloader:
42 repos: &_base-1-repos {}
43 pkgs: &_base-1-pkgs
44 linux-virt:
45 alpine-mirrors:
46 chrony:
47 haveged:
48 nvme-cli:
49 openssh:
50 sudo:
51 tiny-ec2-bootstrap:
52 tzdata:
53 svcs: &_base-1-svcs
54 # sysinit
55 devfs: sysinit
56 dmesg: sysinit
57 hwdrivers: sysinit
58 mdev: sysinit
59 # boot
60 acpid: boot
61 bootmisc: boot
62 haveged: boot
63 hostname: boot
64 hwclock: boot
65 modules: boot
66 swap: boot
67 sysctl: boot
68 syslog: boot
69 # default
70 chronyd:
71 networking:
72 sshd:
73 tiny-ec2-bootstrap:
74 # shutdown
75 killprocs: shutdown
76 mount-ro: shutdown
77 savecache: shutdown
78
diff --git a/profiles/base/current.yaml b/profiles/base/current.yaml
new file mode 100644
index 0000000..7f5e33d
--- /dev/null
+++ b/profiles/base/current.yaml
@@ -0,0 +1,13 @@
1### Current Revision Base Variables
2
3_base-current: &_base-current
4 # Current base revision
5 <<: *_base-1
6 # re-anchor for current
7 repos: &_base-current-repos
8 <<: *_base-1-repos
9 pkgs: &_base-current-pkgs
10 <<: *_base-1-pkgs
11 svcs: &_base-current-svcs
12 <<: *_base-1-svcs
13
diff --git a/profiles/base/edge.yaml b/profiles/base/edge.yaml
new file mode 100644
index 0000000..efd3442
--- /dev/null
+++ b/profiles/base/edge.yaml
@@ -0,0 +1,18 @@
1### Edge Base Variables
2
3_base-edge: &_base-edge
4 # base from current (at the moment)
5 <<: *_base-current
6 # overrides (if any)
7
8 # base repos/pkgs/svcs for edge
9 repos: &_base-edge-repos
10 <<: *_base-current-repos
11 # overrides (if any)
12 pkgs: &_base-edge-pkgs
13 <<: *_base-current-pkgs
14 # overrides (if any)
15 svcs: &_base-edge-svcs
16 <<: *_base-current-svcs
17 # overrides (if any)
18
diff --git a/profiles/test.yaml b/profiles/test.yaml
new file mode 100644
index 0000000..bcb21e1
--- /dev/null
+++ b/profiles/test.yaml
@@ -0,0 +1,48 @@
1### Profile for Testing Builds
2
3# Profile overrides
4test: &test
5 build_region: us-west-2
6 build_subnet: subnet-033a30d7b5220d177
7 ami_regions:
8 - us-east-1
9 - us-west-2
10
11# Build definitions
12builds:
13 test-current-x86_64:
14 <<: *_base-current
15 <<: *_version-current
16 <<: *_x86_64-current
17 repos:
18 <<: *_base-current-repos
19 <<: *_version-current-repos
20 <<: *_x86_64-current-repos
21 pkgs:
22 <<: *_base-current-pkgs
23 <<: *_version-current-pkgs
24 <<: *_x86_64-current-pkgs
25 svcs:
26 <<: *_base-current-svcs
27 <<: *_version-current-svcs
28 <<: *_x86_64-current-svcs
29 <<: *test
30 revision: 'test'
31 test-edge-x86_64:
32 <<: *_base-edge
33 <<: *_version-edge
34 <<: *_x86_64-edge
35 repos:
36 <<: *_base-edge-repos
37 <<: *_version-edge-repos
38 <<: *_x86_64-edge-repos
39 pkgs:
40 <<: *_base-edge-pkgs
41 <<: *_version-edge-pkgs
42 <<: *_x86_64-edge-pkgs
43 svcs:
44 <<: *_base-edge-svcs
45 <<: *_version-edge-svcs
46 <<: *_x86_64-edge-svcs
47 <<: *test
48
diff --git a/profiles/version/3.9.yaml b/profiles/version/3.9.yaml
new file mode 100644
index 0000000..40c2307
--- /dev/null
+++ b/profiles/version/3.9.yaml
@@ -0,0 +1,15 @@
1### Version 3.9 Variables
2
3_version-3.9: &_version-3_9
4 # Versioning
5 version: '3.9'
6 release: '3.9.3'
7 #end_of_life: 'YYYY-MM-DD' # set when known
8
9 # version repos/pkgs/svcs for 3.9
10 repos: &_version-3_9-repos
11 "http://dl-cdn.alpinelinux.org/alpine/v3.9/main":
12 "http://dl-cdn.alpinelinux.org/alpine/v3.9/community":
13 pkgs: &_version-3_9-pkgs {}
14 svcs: &_version-3_9-svcs {}
15
diff --git a/profiles/version/current.yaml b/profiles/version/current.yaml
new file mode 100644
index 0000000..855972c
--- /dev/null
+++ b/profiles/version/current.yaml
@@ -0,0 +1,13 @@
1### Current Version Variables
2
3_version-current: &_version-current
4 # from latest version
5 <<: *_version-3_9
6 # re-anchor for current
7 repos: &_version-current-repos
8 <<: *_version-3_9-repos
9 pkgs: &_version-current-pkgs
10 <<: *_version-3_9-pkgs
11 svcs: &_version-current-svcs
12 <<: *_version-3_9-svcs
13
diff --git a/profiles/version/edge.yaml b/profiles/version/edge.yaml
new file mode 100644
index 0000000..528d56c
--- /dev/null
+++ b/profiles/version/edge.yaml
@@ -0,0 +1,23 @@
1### Edge Version Variables
2
3_version-edge: &_version-edge
4 # from current version
5 <<: *_version-current
6 # overrides
7 version: edge
8 release: edge
9 end_of_life: '@TODAY@'
10 revision: '@NOW@'
11
12 # version repos/pkgs/svcs for edge
13 repos: &_version-edge-repos
14 http://dl-cdn.alpinelinux.org/alpine/edge/main:
15 http://dl-cdn.alpinelinux.org/alpine/edge/community:
16 http://dl-cdn.alpinelinux.org/alpine/edge/testing:
17 pkgs: &_version-edge-pkgs
18 <<: *_version-current-pkgs
19 # overrides (if any)
20 svcs: &_version-edge-svcs
21 <<: *_version-current-svcs
22 # overrides (if any)
23
diff --git a/scripts/make-amis b/scripts/make-amis
new file mode 100755
index 0000000..e0a699e
--- /dev/null
+++ b/scripts/make-amis
@@ -0,0 +1,13 @@
1#!/bin/sh
2
3cd build
4
5# if no build(s) specified, do all profile's builds!
6[ $# -gt 0 ] && BUILDS="$@" || BUILDS=$(ls profile)
7
8for B in $BUILDS
9do
10 BUILD="profile/$B"
11 ( packer build -var-file=$BUILD/vars.json packer.json; echo $? >$BUILD/exit ) | tee $BUILD/output
12 echo "=-> $(cat $BUILD/exit) <-="
13done
diff --git a/nvme/nvme-ebs-links b/scripts/nvme/nvme-ebs-links
index f2c470b..f2c470b 100755
--- a/nvme/nvme-ebs-links
+++ b/scripts/nvme/nvme-ebs-links
diff --git a/nvme/nvme-ebs-mdev.conf b/scripts/nvme/nvme-ebs-mdev.conf
index c30b6fd..c30b6fd 100644
--- a/nvme/nvme-ebs-mdev.conf
+++ b/scripts/nvme/nvme-ebs-mdev.conf
diff --git a/scripts/resolve-profile.py.in b/scripts/resolve-profile.py.in
new file mode 100644
index 0000000..7312957
--- /dev/null
+++ b/scripts/resolve-profile.py.in
@@ -0,0 +1,82 @@
1@PYTHON@
2
3import json, os, shutil, sys, time, yaml
4
5profile = sys.argv[1]
6
7# where we store profile's builds
8profile_dir = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'profile')
9
10# fold these list vars down to scalar
11fold_list_keys = [
12 'ami_access', 'ami_regions'
13]
14
15# parse stdin yaml and resolve refs
16builds = yaml.full_load(sys.stdin.read())['builds']
17
18# clean out any old builds
19if os.path.exists(profile_dir):
20 shutil.rmtree(profile_dir)
21os.makedirs(profile_dir)
22
23# for each build
24for bk, b in builds.items():
25
26 # make profile build directory
27 build_dir = os.path.join(profile_dir, bk)
28 os.makedirs(build_dir)
29 vars_file = os.path.join(build_dir, 'vars.json')
30
31 # populate profile build vars
32 b['profile'] = profile
33 b['profile_build'] = bk
34
35 # edge-related temporal substitutions
36 if b['end_of_life'] == '@TODAY@':
37 b['end_of_life'] = time.strftime('%Y-%m-%d', time.gmtime())
38 if b['revision'] == '@NOW@':
39 b['revision'] = time.strftime('%Y%m%d%H%M%S', time.gmtime())
40
41 # fold list vars to scalars
42 for k in fold_list_keys:
43 if k in b and isinstance(b[k], list):
44 b[k] = ','.join(str(x) for x in b[k])
45
46 # fold 'repos' hash to scalar
47 repos = []
48 for repo, v in b['repos'].items():
49 if v == None: # repo without alias
50 repos.append(repo)
51 elif v != False: # repo with alias (False == skip repo)
52 v.lstrip('@')
53 repos.append(f"@{v} {repo}")
54 b['repos'] = ','.join(str(x) for x in repos)
55
56 # fold 'pkgs' hash to scalar
57 pkgs = []
58 for pkg, v in b['pkgs'].items():
59 if v == None: # unpinned package
60 pkgs.append(pkg)
61 elif v != False: # repo-pinned package (False == skip package)
62 v.lstrip('@')
63 pkgs.append(f'{pkg}@{v}')
64 b['pkgs'] = ','.join(str(x) for x in pkgs)
65
66 # fold 'svcs' hash to scalar
67 svcs = {
68 'sysinit': [],
69 'boot': [],
70 'default': [],
71 'shutdown': []
72 }
73 for svc, v in b['svcs'].items():
74 if v == None: # service in default runlevel
75 svcs['default'].append(svc)
76 elif v != False: # service in specified runlevel (False == skip service)
77 svcs[v].append(svc)
78 b['svcs'] = ':'.join(str(l) + '=' + ','.join(str(s) for s in ss) for l, ss in svcs.items())
79
80 # write build def json file
81 with open(vars_file, 'w') as out:
82 json.dump(b, out, indent=4, separators=(',', ': '))
diff --git a/make_ami.sh b/scripts/setup-ami.sh
index d6c65f7..4102c0f 100755
--- a/make_ami.sh
+++ b/scripts/setup-ami.sh
@@ -3,12 +3,7 @@
3 3
4set -eu 4set -eu
5 5
6MIN_VERSION="3.9" 6# TODO: profile-ize these
7MIN_RELEASE="3.9.0"
8
9: ${VERSION:="${MIN_VERSION}"} # unless otherwise specified
10: ${RELEASE:="${MIN_RELEASE}"} # unless otherwise specified
11
12: ${APK_TOOLS_URI:="https://github.com/alpinelinux/apk-tools/releases/download/v2.10.3/apk-tools-2.10.3-x86_64-linux.tar.gz"} 7: ${APK_TOOLS_URI:="https://github.com/alpinelinux/apk-tools/releases/download/v2.10.3/apk-tools-2.10.3-x86_64-linux.tar.gz"}
13: ${APK_TOOLS_SHA256:="4d0b2cda606720624589e6171c374ec6d138867e03576d9f518dddde85c33839"} 8: ${APK_TOOLS_SHA256:="4d0b2cda606720624589e6171c374ec6d138867e03576d9f518dddde85c33839"}
14: ${ALPINE_KEYS:="http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/alpine-keys-2.1-r1.apk"} 9: ${ALPINE_KEYS:="http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/alpine-keys-2.1-r1.apk"}
@@ -76,25 +71,11 @@ make_filesystem() {
76} 71}
77 72
78setup_repositories() { 73setup_repositories() {
79 local target="$1" # target directory 74 local target="$1" # target directory
80 local add_repos="$2" # extra repo lines, comma separated 75 local repos="$2" # repositories
81 76
82 mkdir -p "$target"/etc/apk/keys 77 mkdir -p "$target"/etc/apk/keys
83 78 echo "$repos" > "$target"/etc/apk/repositories
84 if [ "$VERSION" = 'edge' ]; then
85 cat > "$target"/etc/apk/repositories <<EOF
86http://dl-cdn.alpinelinux.org/alpine/edge/main
87http://dl-cdn.alpinelinux.org/alpine/edge/community
88http://dl-cdn.alpinelinux.org/alpine/edge/testing
89EOF
90 else
91 cat > "$target"/etc/apk/repositories <<EOF
92http://dl-cdn.alpinelinux.org/alpine/v$VERSION/main
93http://dl-cdn.alpinelinux.org/alpine/v$VERSION/community
94EOF
95 fi
96
97 echo "$add_repos" | tr , "\012" >> "$target"/etc/apk/repositories
98} 79}
99 80
100fetch_keys() { 81fetch_keys() {
@@ -132,25 +113,11 @@ setup_chroot() {
132 113
133install_core_packages() { 114install_core_packages() {
134 local target="$1" # target directory 115 local target="$1" # target directory
135 local add_pkgs="$2" # extra packages, space separated 116 local pkgs="$2" # packages, space separated
136 117
137 # Most from: https://git.alpinelinux.org/cgit/alpine-iso/tree/alpine-virt.packages 118 chroot "$target" apk --no-cache add $pkgs
138 #
139 # sudo - to allow alpine user to become root, disallow root SSH logins
140 # tiny-ec2-bootstrap - to bootstrap system from EC2 metadata
141 #
142 chroot "$target" apk --no-cache add \
143 linux-virt \
144 alpine-mirrors \
145 chrony \
146 haveged \
147 nvme-cli \
148 openssh \
149 sudo \
150 tiny-ec2-bootstrap \
151 tzdata \
152 $(echo "$add_pkgs" | tr , ' ')
153 119
120 # TODO: use BOOTSTRAP
154 chroot "$target" apk --no-cache add --no-scripts syslinux 121 chroot "$target" apk --no-cache add --no-scripts syslinux
155 122
156 # Disable starting getty for physical ttys because they're all inaccessible 123 # Disable starting getty for physical ttys because they're all inaccessible
@@ -171,6 +138,7 @@ setup_mdev() {
171 sed -n -i -e '/# fallback/r /tmp/nvme-ebs-mdev.conf' -e 1x -e '2,${x;p}' -e '${x;p}' "$target"/etc/mdev.conf 138 sed -n -i -e '/# fallback/r /tmp/nvme-ebs-mdev.conf' -e 1x -e '2,${x;p}' -e '${x;p}' "$target"/etc/mdev.conf
172} 139}
173 140
141# TODO: use alpine-conf setup-*?
174create_initfs() { 142create_initfs() {
175 local target="$1" 143 local target="$1"
176 144
@@ -186,6 +154,7 @@ create_initfs() {
186 chroot "$target" /sbin/mkinitfs $(basename $(find "$target"/lib/modules/* -maxdepth 0)) 154 chroot "$target" /sbin/mkinitfs $(basename $(find "$target"/lib/modules/* -maxdepth 0))
187} 155}
188 156
157# TODO: use alpine-conf setup-*?
189setup_extlinux() { 158setup_extlinux() {
190 local target="$1" 159 local target="$1"
191 160
@@ -209,6 +178,7 @@ setup_extlinux() {
209 "$target"/etc/update-extlinux.conf 178 "$target"/etc/update-extlinux.conf
210} 179}
211 180
181# TODO: use alpine-conf setup-*?
212install_extlinux() { 182install_extlinux() {
213 local target="$1" 183 local target="$1"
214 184
@@ -239,20 +209,14 @@ EOF
239 209
240enable_services() { 210enable_services() {
241 local target="$1" 211 local target="$1"
242 local add_svcs="$2" 212 local svcs="$2"
243
244 rc_add "$target" default chronyd networking sshd tiny-ec2-bootstrap
245 rc_add "$target" sysinit devfs dmesg hwdrivers mdev
246 rc_add "$target" boot acpid bootmisc haveged hostname hwclock modules swap sysctl syslog
247 rc_add "$target" shutdown killprocs mount-ro savecache
248 213
249 if [ -n "$add_svcs" ]; then 214 local lvl_svcs; for lvl_svcs in $svcs; do
250 local lvl_svcs; for lvl_svcs in $(echo "$add_svcs" | tr : ' '); do 215 rc_add "$target" $(echo "$lvl_svcs" | tr =, ' ')
251 rc_add "$target" $(echo "$lvl_svcs" | tr =, ' ') 216 done
252 done
253 fi
254} 217}
255 218
219# TODO: allow profile to specify alternate ALPINE_USER
256create_alpine_user() { 220create_alpine_user() {
257 local target="$1" 221 local target="$1"
258 222
@@ -312,14 +276,9 @@ version_sorted() {
312} 276}
313 277
314main() { 278main() {
315 [ "$VERSION" != 'edge' ] && { 279 local repos=$(echo "$REPOS" | tr , "\n")
316 version_sorted $MIN_VERSION $VERSION || die "Minimum Alpine version is '$MIN_RELEASE'" 280 local pkgs=$(echo "$PKGS" | tr , ' ')
317 version_sorted $MIN_RELEASE $RELEASE || die "Minimum Alpine release is '$MIN_RELEASE'" 281 local svcs=$(echo "$SVCS" | tr : ' ')
318 }
319
320 local add_repos="$ADD_REPOS"
321 local add_pkgs="$ADD_PKGS"
322 local add_svcs="$ADD_SVCS"
323 282
324 local device="/dev/xvdf" 283 local device="/dev/xvdf"
325 local target="/mnt/target" 284 local target="/mnt/target"
@@ -335,7 +294,7 @@ main() {
335 make_filesystem "$device" "$target" 294 make_filesystem "$device" "$target"
336 295
337 einfo "Configuring Alpine repositories" 296 einfo "Configuring Alpine repositories"
338 setup_repositories "$target" "$add_repos" 297 setup_repositories "$target" "$repos"
339 298
340 einfo "Fetching Alpine signing keys" 299 einfo "Fetching Alpine signing keys"
341 fetch_keys "$target" 300 fetch_keys "$target"
@@ -346,7 +305,7 @@ main() {
346 setup_chroot "$target" 305 setup_chroot "$target"
347 306
348 einfo "Installing core packages" 307 einfo "Installing core packages"
349 install_core_packages "$target" "$add_pkgs" 308 install_core_packages "$target" "$pkgs"
350 309
351 einfo "Configuring and enabling boot loader" 310 einfo "Configuring and enabling boot loader"
352 create_initfs "$target" 311 create_initfs "$target"
@@ -357,7 +316,7 @@ main() {
357 setup_mdev "$target" 316 setup_mdev "$target"
358 setup_fstab "$target" 317 setup_fstab "$target"
359 setup_networking "$target" 318 setup_networking "$target"
360 enable_services "$target" "$add_svcs" 319 enable_services "$target" "$svcs"
361 create_alpine_user "$target" 320 create_alpine_user "$target"
362 configure_ntp "$target" 321 configure_ntp "$target"
363 322
diff --git a/scripts/yaml2json.py.in b/scripts/yaml2json.py.in
new file mode 100644
index 0000000..ec9c8e8
--- /dev/null
+++ b/scripts/yaml2json.py.in
@@ -0,0 +1,9 @@
1@PYTHON@
2
3# Simple little YAML/JSON converter so we can maintain our Packer configs in a
4# sane format that allows comments but also use Packer which only supports JSON
5
6import yaml, json, sys
7
8y = yaml.full_load(open(sys.argv[1]))
9json.dump(y, sys.stdout, indent=4, separators=(',', ': '))
diff --git a/variables.yaml-default b/variables.yaml-default
deleted file mode 100644
index 0cd3cc2..0000000
--- a/variables.yaml-default
+++ /dev/null
@@ -1,77 +0,0 @@
1### Builder-Instance Options ###
2
3# Region to build in, if we initiate a build from outside AWS
4region:
5
6# Subnet ID in which the builder instance is to be launched. VPC will be
7# automatically determined.
8subnet:
9
10# Optional security group to apply to the builder instance
11security_group:
12
13# By default, public IPs are assigned (or not) per the subnet's configuration.
14# Set to "true" or "false" to explicitly override the subnet's public IP auto-
15# assign configuration.
16public_ip: ""
17
18
19### Build Options ###
20
21# Uncomment/increment every for every rebuild of an Alpine release;
22# re-comment/zero for every new Alpine release
23#revision: "-1"
24
25# AMI name prefix and suffix
26ami_name_prefix: "alpine-ami-"
27ami_name_suffix: ""
28
29# AMI description prefix and suffix
30ami_desc_prefix: "Alpine Linux "
31ami_desc_suffix: " - https://github.com/mcrute/alpine-ec2-ami"
32
33# List of custom lines to add to /etc/apk/repositories
34add_repos:
35# - "@my-repo http://my-repo.tld/path"
36
37# List of additional packages to add to the AMI.
38add_pkgs:
39# - package-name
40
41# Additional services to start at the specified level
42add_svcs:
43# boot:
44# - service1
45# default:
46# - service2
47
48# Size of the AMI image (in GiB).
49volume_size: "1"
50
51# Encrypt the AMI?
52encrypt_ami: "false"
53
54# List of groups that should have access to the AMI. However, only two
55# values are currently supported: 'all' for public, '' or unset for private.
56ami_access:
57 - "all"
58
59# List of regions to where the AMI should be copied
60deploy_regions:
61 - "us-east-1"
62 - "us-east-2"
63 - "us-west-1"
64 - "us-west-2"
65 - "ca-central-1"
66 - "eu-central-1"
67 - "eu-north-1"
68 - "eu-west-1"
69 - "eu-west-2"
70 - "eu-west-3"
71 - "ap-northeast-1"
72 - "ap-northeast-2"
73# - "ap-northeast-3" # skipped, available by subscription only
74 - "ap-southeast-1"
75 - "ap-southeast-2"
76 - "ap-south-1"
77 - "sa-east-1"
© 2015 Mike Crute