-
Notifications
You must be signed in to change notification settings - Fork 83
/
Copy pathchange-yaml.sh
executable file
·101 lines (84 loc) · 3.4 KB
/
change-yaml.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
#!/bin/bash
function usage() {
echo
echo "Changes the image.*.yaml file and adds it to the current commit (git add)"
echo
echo "Usage: change-yaml.sh [options] SPARK_VERSION"
echo
echo "required arguments"
echo
echo " SPARK_VERSION The spark version number, like 3.0.0"
echo
echo "optional arguments:"
echo
echo " -h Show this message"
}
# Set the hadoop version
HVER=3.2
while getopts h opt; do
case $opt in
h)
usage
exit 0
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
esac
done
shift "$((OPTIND-1))"
if [ "$#" -lt 1 ]; then
echo No spark version specified
usage
exit 1
fi
SPARK=$1
# Extract the current spark version from the image.yaml file
# Works by parsing the line following "name: sparkversion"
VER=$(sed -n '\@name: sparkversion@!b;n;p' image.yaml | tr -d '[:space:]' | cut -d':' -f2)
if [ "$VER" == "$SPARK" ]; then
echo "Nothing to do, spark version in image.yaml is already $SPARK"
exit 0
fi
# Change spark distro and download urls
if [ ! -z ${SPARK+x} ]; then
# TODO remove this download when sha512 support lands in upstream cekit (elmiko)
if [ -f "/tmp/spark-${SPARK}-bin-hadoop${HVER}.tgz" ]; then
echo
echo Using existing "/tmp/spark-${SPARK}-bin-hadoop${HVER}.tgz", if this is not what you want delete it and run again
echo
else
wget https://archive.apache.org/dist/spark/spark-${SPARK}/spark-${SPARK}-bin-hadoop${HVER}.tgz -O /tmp/spark-${SPARK}-bin-hadoop${HVER}.tgz
if [ "$?" -ne 0 ]; then
echo "Failed to download the specified version Spark archive"
exit 1
fi
fi
wget https://archive.apache.org/dist/spark/spark-${SPARK}/spark-${SPARK}-bin-hadoop${HVER}.tgz.sha512 -O /tmp/spark-${SPARK}-bin-hadoop${HVER}.tgz.sha512
if [ "$?" -ne 0 ]; then
echo "Failed to download the sha512 sum for the specified Spark version"
exit 1
fi
# TODO remove this checksum calculation when sha512 support lands in upstream cekit (elmiko)
calcsum=$(sha512sum /tmp/spark-${SPARK}-bin-hadoop${HVER}.tgz | cut -d" " -f1)
sum=$(cat /tmp/spark-${SPARK}-bin-hadoop${HVER}.tgz.sha512 | tr -d [:space:] | cut -d: -f2 | tr [:upper:] [:lower:])
if [ "$calcsum" != "$sum" ]; then
echo "Failed to confirm authenticity of Spark archive, checksum mismatch"
echo "sha512sum : ${calcsum}"
echo ".sha512 file: ${sum}"
exit 1
fi
# Fix the url references
sed -i "s@https://archive\.apache\.org/dist/spark/spark-.*/spark-.*-bin-hadoop.*\.tgz@https://archive\.apache\.org/dist/spark/spark-${SPARK}/spark-${SPARK}-bin-hadoop${HVER}\.tgz@" modules/spark/module.yaml
# TODO replace this with sha512 when it lands in upstream cekit (elmiko)
# Fix the md5 sum references on the line following the url
calcsum=$(md5sum /tmp/spark-${SPARK}-bin-hadoop${HVER}.tgz | cut -d" " -f1)
sed -i '\@url: https://archive.apache.org/dist/spark/@!b;n;s/md5.*/md5: '$calcsum'/' modules/spark/module.yaml
# Fix the spark version label
sed -i '\@name: sparkversion@!b;n;s/value.*/value: '$SPARK'/' image.yaml
# Fix the image version value (do this for incomplete as well)
V=$(echo $SPARK | cut -d'.' -f1,2)
sed -i 's@^version:.*-latest$@version: '$V'-latest@' image*.yaml
fi
git add image.yaml