From acf0e6687010b26b39fa5b658dc13d6413aa87d7 Mon Sep 17 00:00:00 2001
From: odidev <odidev@puresoftware.com>
Date: Fri, 7 Aug 2020 13:43:38 +0000
Subject: [PATCH] Added support for multiplatform hadoop-lzo jar

- linux_multiarch profile added in pom.xml to combine jar for x86 and aarch64
- docker-compose.yaml and Dockerfile added for cross-compilation support
- updated config.sub to added aarch64 support
- added README.md having instructions to create a single jar for x86 and aarch64 both
---
 docker/Dockerfile                 |  39 ++++++++++
 docker/README.md                  |  13 ++++
 docker/docker-compose.yaml        |  23 ++++++
 pom.xml                           | 122 ++++++++++++++++++++++++++++++
 src/main/native/config/config.sub |   4 +-
 5 files changed, 199 insertions(+), 2 deletions(-)
 create mode 100644 docker/Dockerfile
 create mode 100644 docker/README.md
 create mode 100644 docker/docker-compose.yaml

diff --git a/docker/Dockerfile b/docker/Dockerfile
new file mode 100644
index 00000000..4febf9b9
--- /dev/null
+++ b/docker/Dockerfile
@@ -0,0 +1,39 @@
+FROM centos:7.6.1810
+
+ARG gcc_version=4.9-2016.02
+ENV LZO_VERSION=2.10
+ENV GCC_VERSION $gcc_version
+
+# Install requirements
+RUN yum install -y wget tar git make java-1.8.0-openjdk-devel maven which gcc
+
+# Install aarch64 gcc toolchain
+RUN set -x && \
+    wget https://releases.linaro.org/components/toolchain/binaries/$GCC_VERSION/aarch64-linux-gnu/gcc-linaro-$GCC_VERSION-x86_64_aarch64-linux-gnu.tar.xz && \
+    tar xvf gcc-linaro-$GCC_VERSION-x86_64_aarch64-linux-gnu.tar.xz
+
+# Settting paths
+ENV PATH="/gcc-linaro-$GCC_VERSION-x86_64_aarch64-linux-gnu/bin:${PATH}"
+ENV JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk/"
+
+# Checkout and build lzo for aarch64
+ADD https://www.oberhumer.com/opensource/lzo/download/lzo-$LZO_VERSION.tar.gz ./
+RUN tar -xvf lzo-$LZO_VERSION.tar.gz && \
+    cd lzo-$LZO_VERSION/ && \
+    ./configure --enable-shared --prefix /usr/local/lzo-$LZO_VERSION-arm --host=aarch64-linux-gnu && \
+    make clean && \
+    make install
+
+# Installing cross compiled lzo on aarch64 gcc toolchain
+RUN cp -r  /usr/local/lzo-$LZO_VERSION-arm/lib/* /gcc-linaro-$GCC_VERSION-x86_64_aarch64-linux-gnu/aarch64-linux-gnu/lib64/ && \
+    cp -r  /usr/local/lzo-$LZO_VERSION-arm/include/* /gcc-linaro-$GCC_VERSION-x86_64_aarch64-linux-gnu/aarch64-linux-gnu/include/
+
+# Build lzo for x86
+RUN cd /lzo-$LZO_VERSION/ && \
+    ./configure --enable-shared --prefix /usr/local/lzo-$LZO_VERSION && \
+    make clean && \
+    make install
+
+# Settting lzo paths
+ENV C_INCLUDE_PATH="/usr/local/lzo-$LZO_VERSION/include"
+ENV LIBRARY_PATH="/usr/local/lzo-$LZO_VERSION/lib"
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 00000000..e640946f
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,13 @@
+# Using the docker image
+
+```
+cd /path/to/hadoop-lzo/
+```
+
+## Create a jar having amd64 and aarch64 libraries for hadoop-lzo
+
+```
+docker-compose -f docker/docker-compose.yaml run cross_compile
+```
+The default version of aarch64 gcc is `4.9-2016.02` and lzo is `2.10`. 
+Update the parameter in `docker-compose.yaml` to use a version you want.
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
new file mode 100644
index 00000000..30b50418
--- /dev/null
+++ b/docker/docker-compose.yaml
@@ -0,0 +1,23 @@
+version: "3"
+
+services:
+
+  cross-compile-runtime-setup:
+    image: commons:cross_compile_aarch64
+    build:
+      context: .
+      dockerfile: Dockerfile 
+      args:
+        gcc_version : "4.9-2016.02"
+
+  cross-compile:
+    image: commons:cross_compile_aarch64
+    depends_on: [cross-compile-runtime-setup]
+    volumes:
+      - ~/.ssh:/root/.ssh:delegated
+      - ~/.gnupg:/root/.gnupg:delegated
+      - ..:/code:delegated
+      - ~/.m2:/root/.m2:delegated
+    # Since we are cross compiling hadoop-lzo which cannot be loaded on x86_64, we add `skipTests` here to skip the test.
+    command: /bin/bash -cl "mvn clean install -e -Plinux-multiarch -DskipTests"
+    working_dir: /code
diff --git a/pom.xml b/pom.xml
index 4689a25e..ae9beffc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,6 +133,128 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+       <id>linux-multiarch</id>
+       <properties>
+          <exe.armcompiler>aarch64-linux-gnu-gcc</exe.armcompiler>
+          <exe.amdcompiler>x86_64-redhat-linux-gcc</exe.amdcompiler>
+       </properties>
+       <activation>
+          <activeByDefault>false</activeByDefault>
+       </activation>
+       <dependencies>
+          <dependency>
+             <groupId>org.apache.hadoop</groupId>
+             <artifactId>hadoop-common</artifactId>
+             <version>${hadoop.current.version}</version>
+             <scope>provided</scope>
+          </dependency>
+          <dependency>
+             <groupId>org.apache.hadoop</groupId>
+             <artifactId>hadoop-mapreduce-client-core</artifactId>
+             <version>${hadoop.current.version}</version>
+             <scope>provided</scope>
+          </dependency>
+       </dependencies>
+       <build>
+          <plugins>
+             <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                   <execution>
+                      <id>set-props-non-win</id>
+                      <phase>initialize</phase>
+                      <goals>
+                         <goal>run</goal>
+                      </goals>
+                      <configuration>
+                         <exportAntProperties>true</exportAntProperties>
+                         <target name="set-props-non-win" description="sets key properties that are used throughout" if="non-windows">
+                            <exec executable="sed" inputstring="${os.name}" outputproperty="nonspace.os">
+                               <arg value="s/ /_/g" />
+                            </exec>
+                            <property name="build.armplatform" value="${nonspace.os}-aarch64-${sun.arch.data.model}" />
+                            <property name="build.armnative" value="${project.build.directory}/native/${build.armplatform}" />
+                            <property name="build.armnative.target" value="${project.build.outputDirectory}/native/${build.armplatform}" />
+                            <property name="build.amdplatform" value="${nonspace.os}-amd64-${sun.arch.data.model}" />
+                            <property name="build.amdnative" value="${project.build.directory}/native/${build.amdplatform}" />
+                            <property name="build.amdnative.target" value="${project.build.outputDirectory}/native/${build.amdplatform}" />
+		            <property name="native.src.dir" value="${basedir}/src/main/native" />
+                            <property name="test.build.dir" value="${project.build.directory}/test-classes" />
+                            <property name="test.log.dir" value="${test.build.dir}/logs" />
+                         </target>
+                      </configuration>
+                   </execution>
+                   <execution>
+                      <id>build-native-non-win</id>
+                      <phase>compile</phase>
+                      <goals>
+                         <goal>run</goal>
+                      </goals>
+                      <configuration>
+                         <target name="build-native-non-win" if="non-windows" unless="native.uptodate" description="compiles native code">
+                            <property name="make.cmd" value="make" />
+                            <property name="build.classes" refid="maven.compile.classpath" />
+                            <mkdir dir="${build.armnative}/lib" />
+                            <mkdir dir="${build.armnative.target}/lib" />
+                            <mkdir dir="${build.armnative}/src/com/hadoop/compression/lzo" />
+                            <mkdir dir="${build.amdnative}/lib" />
+                            <mkdir dir="${build.amdnative.target}/lib" />
+                            <mkdir dir="${build.amdnative}/src/com/hadoop/compression/lzo" />
+                            <javah classpath="${build.classes}" destdir="${build.armnative}/src/com/hadoop/compression/lzo" force="yes" verbose="yes">
+                               <class name="com.hadoop.compression.lzo.LzoCompressor" />
+                               <class name="com.hadoop.compression.lzo.LzoDecompressor" />
+                            </javah>
+                            <exec dir="${build.armnative}" executable="sh" failonerror="true">
+                               <env key="CC" value="${exe.armcompiler}" />
+                               <env key="OS_NAME" value="${os.name}" />
+                               <env key="OS_ARCH" value="aarch64" />
+                               <env key="LDFLAGS" value="-Wl,--no-as-needed -lrt" />
+                               <env key="JVM_DATA_MODEL" value="${sun.arch.data.model}" />
+                               <env key="NATIVE_SRCDIR" value="${native.src.dir}" />
+                               <arg line="${native.src.dir}/configure --host=aarch64-linux-gnu" />
+                            </exec>
+                            <exec dir="${build.armnative}" executable="${make.cmd}" failonerror="true">
+                               <env key="OS_NAME" value="${os.name}" />
+                               <env key="OS_ARCH" value="aarch64" />
+                               <env key="JVM_DATA_MODEL" value="${sun.arch.data.model}" />
+                               <env key="HADOOP_NATIVE_SRCDIR" value="${native.src.dir}" />
+                            </exec>
+                            <exec dir="${build.armnative}" executable="sh" failonerror="true">
+                               <arg line="${build.armnative}/libtool --mode=install cp ${build.armnative}/libgplcompression.la ${build.armnative}/lib" />
+                            </exec>
+                            <exec dir="${build.amdnative}" executable="sh" failonerror="true">
+                               <env key="CC" value="${exe.amdcompiler}" />
+                               <env key="OS_NAME" value="${os.name}" />
+                               <env key="OS_ARCH" value="amd64" />
+                               <env key="LDFLAGS" value="-Wl,--no-as-needed -lrt" />
+                               <env key="JVM_DATA_MODEL" value="${sun.arch.data.model}" />
+                               <env key="NATIVE_SRCDIR" value="${native.src.dir}" />
+                               <arg line="${native.src.dir}/configure" />
+                            </exec>
+                            <exec dir="${build.amdnative}" executable="${make.cmd}" failonerror="true">
+                               <env key="OS_NAME" value="${os.name}" />
+                               <env key="OS_ARCH" value="aarch64" />
+                               <env key="JVM_DATA_MODEL" value="${sun.arch.data.model}" />
+                               <env key="HADOOP_NATIVE_SRCDIR" value="${native.src.dir}" />
+                            </exec>
+                            <exec dir="${build.amdnative}" executable="sh" failonerror="true">
+                               <arg line="${build.amdnative}/libtool --mode=install cp ${build.amdnative}/libgplcompression.la ${build.amdnative}/lib" />
+                            </exec>
+                            <copy todir="${build.armnative.target}/lib">
+                               <fileset dir="${build.armnative}/lib" />
+                            </copy>
+                           <copy todir="${build.amdnative.target}/lib">
+                               <fileset dir="${build.amdnative}/lib" />
+                            </copy>
+                         </target>
+                      </configuration>
+                   </execution>
+                </executions>
+             </plugin>
+          </plugins>
+       </build>
+    </profile>
   </profiles>
 
   <build>
diff --git a/src/main/native/config/config.sub b/src/main/native/config/config.sub
index d8fd2f8f..871eb196 100755
--- a/src/main/native/config/config.sub
+++ b/src/main/native/config/config.sub
@@ -230,7 +230,7 @@ case $basic_machine in
 	| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
 	| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
 	| am33_2.0 \
-	| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \
+	| arc | arm | aarch64* | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \
 	| c4x | clipper \
 	| d10v | d30v | dlx | dsp16xx \
 	| fr30 | frv \
@@ -296,7 +296,7 @@ case $basic_machine in
 	| alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
 	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
 	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
-	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
+	| arm-*  | armbe-* | aarch64* | armle-* | armeb-* | armv*-* \
 	| avr-* \
 	| bs2000-* \
 	| c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \