From d560038342ad45fabd10be85d8343d05d7f20c92 Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Sun, 7 Mar 2021 13:10:27 -0800 Subject: Add script to build an ARM binary on AWS. --- tools/build-arm.sh | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tools/build-arm.sh (limited to 'tools/build-arm.sh') diff --git a/tools/build-arm.sh b/tools/build-arm.sh new file mode 100644 index 000000000..a9a794ef6 --- /dev/null +++ b/tools/build-arm.sh @@ -0,0 +1,82 @@ +#!/bin/sh + +# Spin up an ARM build machine using aws cli, build pandoc, and +# download the artifact. +# +# We need to use us-east-2; since my us-west-1 has EC2-classic. +# docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-classic-platform.html + +aws configure set default.region us-east-2 + +# Now start instance with volume 16GB. +# note, the AMI id is different for different regions. +# for us east-2 the Debian Buster ARM AMI is ami-0fa8979d18f69948b: + +echo "Creating instance..." + +aws ec2 run-instances --image-id ami-0fa8979d18f69948b --count 1 --instance-type t4g.2xlarge --block-device-mapping 'DeviceName=/dev/xvda,Ebs={VolumeSize=16}' --key-name debian-arm-us-east-2 --security-group-ids sg-086ffbadc286c5c00 > ec2.json + +# Now get the public IP address. + +INSTANCEID=$(jq '.Instances[0].InstanceId' ec2.json | sed -e 's/"//g') +IPADDR=$(aws ec2 describe-instances --instance-ids="$INSTANCEID" --query 'Reservations[0].Instances[0].PublicIpAddress' | sed -e 's/"//g') + +clean_up() { + echo "Terminating the instance..." + aws ec2 terminate-instances --instance-ids "$INSTANCEID" +} +trap clean_up EXIT + +echo "Waiting for instance to start up..." + +STATUS=none +while [ "$STATUS" != "running" ] +do + sleep 20 + STATUS=$(aws ec2 describe-instance-status --instance-id "$INSTANCEID" | jq '.InstanceStatuses[0].InstanceState.Name' | sed -e 's/"//g') + echo "...$STATUS" +done + +# At this point you can connect via SSH, or run this script: +# $ ssh -i ~/.ssh/debian-arm-us-east-2.pem admin@$IPADDR + +SSH="ssh -i ~/.ssh/debian-arm-us-east-2.pem admin@$IPADDR" + +echo "Provisioning..." + +$SSH < Date: Sun, 7 Mar 2021 13:27:11 -0800 Subject: build-arm.sh: retrieve build artifacts after success. --- tools/build-arm.sh | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'tools/build-arm.sh') diff --git a/tools/build-arm.sh b/tools/build-arm.sh index a9a794ef6..7bd04477c 100644 --- a/tools/build-arm.sh +++ b/tools/build-arm.sh @@ -41,6 +41,7 @@ done # $ ssh -i ~/.ssh/debian-arm-us-east-2.pem admin@$IPADDR SSH="ssh -i ~/.ssh/debian-arm-us-east-2.pem admin@$IPADDR" +SCP="scp -r -i ~/.ssh/debian-arm-us-east-2.pem admin@$IPADDR:" echo "Provisioning..." @@ -79,4 +80,10 @@ do $SSH "ls -l linux/artifacts/*.tar.gz" && break done +# Retrieve the artifacts +echo "Successful build. Retrieving artifacts..." + +$SCP -r src/pandoc/linux/artifacts "arm-build-artifacts-$(date +%s)" + +exit 0 -- cgit v1.2.3 From 7eb4662df46b856e6e332150bbb21a17d0ee988b Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Sun, 7 Mar 2021 18:05:22 -0800 Subject: Fixed build-arm script. Use environment variables which can be set to use different AMIs, keys, or instance types. Correctly detect successful build. --- tools/build-arm.sh | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) (limited to 'tools/build-arm.sh') diff --git a/tools/build-arm.sh b/tools/build-arm.sh index 7bd04477c..427bb21ea 100644 --- a/tools/build-arm.sh +++ b/tools/build-arm.sh @@ -1,5 +1,10 @@ #!/bin/sh +IMAGE_ID=ami-0fa8979d18f69948b +INSTANCE_TYPE=t4g.2xlarge +KEY_NAME=debian-arm-us-east-2 +SECURITY_GROUP_ID=sg-086ffbadc286c5c00 + # Spin up an ARM build machine using aws cli, build pandoc, and # download the artifact. # @@ -8,22 +13,23 @@ aws configure set default.region us-east-2 -# Now start instance with volume 16GB. -# note, the AMI id is different for different regions. -# for us east-2 the Debian Buster ARM AMI is ami-0fa8979d18f69948b: - echo "Creating instance..." -aws ec2 run-instances --image-id ami-0fa8979d18f69948b --count 1 --instance-type t4g.2xlarge --block-device-mapping 'DeviceName=/dev/xvda,Ebs={VolumeSize=16}' --key-name debian-arm-us-east-2 --security-group-ids sg-086ffbadc286c5c00 > ec2.json +aws ec2 run-instances --image-id "$IMAGE_ID" --count 1 --instance-type "$INSTANCE_TYPE" --block-device-mapping 'DeviceName=/dev/xvda,Ebs={VolumeSize=16}' --key-name "$KEY_NAME" --security-group-ids "$SECURITY_GROUP_ID" > ec2.json + +jq < ec2.json # Now get the public IP address. INSTANCEID=$(jq '.Instances[0].InstanceId' ec2.json | sed -e 's/"//g') IPADDR=$(aws ec2 describe-instances --instance-ids="$INSTANCEID" --query 'Reservations[0].Instances[0].PublicIpAddress' | sed -e 's/"//g') +echo "IP address is $IPADDR" + clean_up() { - echo "Terminating the instance..." - aws ec2 terminate-instances --instance-ids "$INSTANCEID" + echo "Terminating the instance in 20 seconds..." + echo "Ctrl-C to preserve it." + sleep 20 && aws ec2 terminate-instances --instance-ids "$INSTANCEID" } trap clean_up EXIT @@ -40,8 +46,7 @@ done # At this point you can connect via SSH, or run this script: # $ ssh -i ~/.ssh/debian-arm-us-east-2.pem admin@$IPADDR -SSH="ssh -i ~/.ssh/debian-arm-us-east-2.pem admin@$IPADDR" -SCP="scp -r -i ~/.ssh/debian-arm-us-east-2.pem admin@$IPADDR:" +SSH="ssh -i ~/.ssh/$KEY_NAME.pem admin@$IPADDR" echo "Provisioning..." @@ -74,16 +79,17 @@ EOF while true do - sleep 60 - $SSH "tail -n1 src/pandoc/docker.log && free -h" + sleep 20 + # print last line of log output and free memory + $SSH "tail -n1 src/pandoc/docker.log && free -h | grep Mem" # Check to see if the artifact has been produced - $SSH "ls -l linux/artifacts/*.tar.gz" && break + $SSH "ls -l src/pandoc/linux/artifacts/*.tar.gz 2>/dev/null" && break done # Retrieve the artifacts echo "Successful build. Retrieving artifacts..." -$SCP -r src/pandoc/linux/artifacts "arm-build-artifacts-$(date +%s)" +scp -i "$HOME/.ssh/$KEY_NAME.pem" -r "admin@$IPADDR:src/pandoc/linux/artifacts" "arm-build-artifacts-$(date +%s)" exit 0 -- cgit v1.2.3 From f8e848094c7fa3eeb3c391ebce8944492671c0da Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Sun, 7 Mar 2021 19:02:36 -0800 Subject: build-arm.sh: print total build time --- tools/build-arm.sh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tools/build-arm.sh') diff --git a/tools/build-arm.sh b/tools/build-arm.sh index 427bb21ea..c9ee33e20 100644 --- a/tools/build-arm.sh +++ b/tools/build-arm.sh @@ -92,4 +92,7 @@ echo "Successful build. Retrieving artifacts..." scp -i "$HOME/.ssh/$KEY_NAME.pem" -r "admin@$IPADDR:src/pandoc/linux/artifacts" "arm-build-artifacts-$(date +%s)" +# Let's see how long this took! +uptime + exit 0 -- cgit v1.2.3 From 459085c64282ac6084ecc6c35ac942d1eae5d3fe Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Sun, 7 Mar 2021 20:55:52 -0800 Subject: build-arm.sh: env variable for ARTIFACTS. --- tools/build-arm.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'tools/build-arm.sh') diff --git a/tools/build-arm.sh b/tools/build-arm.sh index c9ee33e20..85ebc47d8 100644 --- a/tools/build-arm.sh +++ b/tools/build-arm.sh @@ -4,6 +4,7 @@ IMAGE_ID=ami-0fa8979d18f69948b INSTANCE_TYPE=t4g.2xlarge KEY_NAME=debian-arm-us-east-2 SECURITY_GROUP_ID=sg-086ffbadc286c5c00 +ARTIFACTS="${ARTIFACTS:-build-artifacts-$(date +%s)}" # Spin up an ARM build machine using aws cli, build pandoc, and # download the artifact. @@ -90,7 +91,10 @@ done echo "Successful build. Retrieving artifacts..." -scp -i "$HOME/.ssh/$KEY_NAME.pem" -r "admin@$IPADDR:src/pandoc/linux/artifacts" "arm-build-artifacts-$(date +%s)" +scp -i "$HOME/.ssh/$KEY_NAME.pem" -r "admin@$IPADDR:src/pandoc/linux/artifacts" "$ARTIFACTS" + +echo "Artifacts saved in $ARTIFACTS" +ls "$ARTIFACTS" # Let's see how long this took! uptime -- cgit v1.2.3 From 7683912591836040c02ec9c25d8207116882e27d Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Mon, 8 Mar 2021 11:57:52 -0800 Subject: build-arm.sh: fix elapsed time. --- tools/build-arm.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'tools/build-arm.sh') diff --git a/tools/build-arm.sh b/tools/build-arm.sh index 85ebc47d8..e413e5437 100644 --- a/tools/build-arm.sh +++ b/tools/build-arm.sh @@ -6,6 +6,8 @@ KEY_NAME=debian-arm-us-east-2 SECURITY_GROUP_ID=sg-086ffbadc286c5c00 ARTIFACTS="${ARTIFACTS:-build-artifacts-$(date +%s)}" +STARTTIME=$(date +%H:%M) + # Spin up an ARM build machine using aws cli, build pandoc, and # download the artifact. # @@ -96,7 +98,9 @@ scp -i "$HOME/.ssh/$KEY_NAME.pem" -r "admin@$IPADDR:src/pandoc/linux/artifacts" echo "Artifacts saved in $ARTIFACTS" ls "$ARTIFACTS" -# Let's see how long this took! -uptime +ENDTIME=$(date +%H:%M) + +echo "Started: $STARTTIME" +echo "Finished: $ENDTIME" exit 0 -- cgit v1.2.3 From cac796e1ab4ed170658d9056b27368e30cde3fcb Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Mon, 8 Mar 2021 14:51:03 -0800 Subject: ARM build script: more reliable detection of completion. Previously we downloaded the tar.gz before it was complete. --- linux/make_artifacts.sh | 9 +++++++++ tools/build-arm.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'tools/build-arm.sh') diff --git a/linux/make_artifacts.sh b/linux/make_artifacts.sh index b32384ed4..43002218e 100644 --- a/linux/make_artifacts.sh +++ b/linux/make_artifacts.sh @@ -11,6 +11,13 @@ esac ARTIFACTS="${ARTIFACTS:-/artifacts}" +rm $ARTIFACTS/* + +clean_up() { + echo "All done!" > "$ARTIFACTS/DONE" +} +trap clean_up EXIT + # build binaries cabal --version @@ -72,3 +79,5 @@ gzip -9 $TARGET/share/man/man1/pandoc.1 tar cvzf $TARGET-linux-$ARCHITECTURE.tar.gz $TARGET rm -r $TARGET + +exit 0 diff --git a/tools/build-arm.sh b/tools/build-arm.sh index e413e5437..dc13e1435 100644 --- a/tools/build-arm.sh +++ b/tools/build-arm.sh @@ -86,7 +86,7 @@ do # print last line of log output and free memory $SSH "tail -n1 src/pandoc/docker.log && free -h | grep Mem" # Check to see if the artifact has been produced - $SSH "ls -l src/pandoc/linux/artifacts/*.tar.gz 2>/dev/null" && break + $SSH "ls -l src/pandoc/linux/artifacts/DONE 2>/dev/null" && break done # Retrieve the artifacts -- cgit v1.2.3 From 0012750c421b74313ad950f95730d6ee21c08408 Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Thu, 18 Mar 2021 10:22:49 -0700 Subject: build-arm.sh : remove strict checking on first ssh access --- tools/build-arm.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'tools/build-arm.sh') diff --git a/tools/build-arm.sh b/tools/build-arm.sh index dc13e1435..0a14c7302 100644 --- a/tools/build-arm.sh +++ b/tools/build-arm.sh @@ -49,6 +49,8 @@ done # At this point you can connect via SSH, or run this script: # $ ssh -i ~/.ssh/debian-arm-us-east-2.pem admin@$IPADDR +ssh -o "StrictHostKeyChecking=no" -i "~/.ssh/$KEY_NAME.pem" admin@$IPADDR uname -a + SSH="ssh -i ~/.ssh/$KEY_NAME.pem admin@$IPADDR" echo "Provisioning..." -- cgit v1.2.3 From 40a78ea05d7cf6f7b1e7b2ae7054fed1e5aa6772 Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Sat, 3 Jul 2021 11:38:04 -0700 Subject: Upgrade Debian 10 AMI for build-arm.sh. --- tools/build-arm.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tools/build-arm.sh') diff --git a/tools/build-arm.sh b/tools/build-arm.sh index 0a14c7302..dc275e4b2 100644 --- a/tools/build-arm.sh +++ b/tools/build-arm.sh @@ -1,6 +1,8 @@ #!/bin/sh -IMAGE_ID=ami-0fa8979d18f69948b +#old version: +#IMAGE_ID=ami-0fa8979d18f69948b +IMAGE_ID=ami-0cd6b53a434812702 INSTANCE_TYPE=t4g.2xlarge KEY_NAME=debian-arm-us-east-2 SECURITY_GROUP_ID=sg-086ffbadc286c5c00 -- cgit v1.2.3