WIP: first stab at working ISPC support

alichnewsky · Oct 18, 2016 · 3aaf8ac · 3aaf8ac
1 parent ef6d620
commit 3aaf8ac
Show file tree

Hide file tree

Showing 6 changed files with 303 additions and 33 deletions.
diff --git a/README.md b/README.md
@@ -1,22 +1,85 @@
 LRTE
 ====
 
-Fork of https://code.google.com/p/google-search-appliance-mirror/downloads/detail?name=grte-1.2.2-src.tar.bz2&amp;can=2&amp;q=
-
-Original source code downloaded from:
-
-https://google-search-appliance-mirror.googlecode.com/files/grte-1.2.2-src.tar.bz2
-https://google-search-appliance-mirror.googlecode.com/files/crosstoolv13-gcc-4.4.0-glibc-2.3.6-grte-1.0-36185.src.rpm
-https://google-search-appliance-mirror.googlecode.com/files/grte-python2.4-2.4.6-7.src.tar.bz2
+Fork of https://github.com/mzhaom/lrte
 
 ChangeLog
 =========
 
- - grte-1.2.2-src.tar.bz2 -> grte with sources stripped
+ - still using Google's GCC 4.9.2 branch and Clang 4.0 dev branches
+ - clang includes openmp and polly
+ - added a build of Intel's [https://github.com/ispc/ispc/][ispc]
+
+
+Install
+=======
+
+install precompiled lrtev2 package. On production machine, lrtev2-runtime is all you need.
+Debug symbols, includes, etc are all optional
+
+to install crosstools v2 on debian :
+```
+sudo dpkg -i output/lrte/results/debs/lrtev2-runtime*.deb
+
+sudo dpkg -i output/lrte/results/debs/lrtev2-crosstoolv2-gcc-4.9*.deb
+sudo dpkg -i output/lrte/results/debs/lrtev2-crosstoolv2-clang-4.0*.deb
+sudo dpkg -i output/lrte/results/debs/lrtev2-crosstoolv2-ispc-1.9.2dev*.deb
+
+```
+the same thing can indeed be done with RPMs
+
+Build:
+======
+I build all of this with a GCE VM using debian:jessie docker images. ubuntu:14.04 should work as well. and may have more reliable APT mirrors.
+
+Install LRTE
+```
+git clone https://github.com/alichnewsky/lrte.git
+cd lrte
+TAR_DIR=upstream ./grte/downloads.sh
+
+./release.py  --no_build_crosstool --email [email protected] --docker_image=debian:jessie
+```
+
+Download gcc, clang and ispc crosstool source code
 
+```
+bash crosstool/get-gcc-svn.sh
+bash crosstool/get-clang-svn.sh
+bash crosstool/get-ispc-from-github.sh
+```
 
-Install and Usage
-=================
+Build and install crosstools
+```
+./release.py  --no_build_lrte --email [email protected] --docker_image=debian:jessie
+
+```
+
+Only building ispc when lrte, `gcc` and `clang` crosstools have already been built
+```
+
+./release.py  --no_build_lrte --crosstool_skip gcc clang --email [email protected] --docker_image=debian:jessie
+
+```
+
+The following part contain SVN or git version numbers and care should be taken when doing updates:
+-
+- 
+
+Releasing to GitHub
+===================
+AFAIK releases still have to be created manually from the github website
+I could not get them created programmatically
+
+Assuming you have an un-encrypted .netrc, and that release v2.2_0 has been created in github, and that you have installed github-release tool(https://github.com/aktau/github-release) in you rpath:
+
+```
+export GITHUB_USER=username GITHUB_TOKEN=$(cat ~/.netrc  |egrep "^password" | awk '{print $2}' | uniq) RELEASE_TAG=v2.2_0
+bash upload-github-release.sh $(find output/lrte/results/ -type f)
+```
+
+Original Install and Usage notes
+================================
 
 It's recommended to just install the precomiled packages from release
 page. For example, to install lrtev2 with crosstool v2:
@@ -60,4 +123,12 @@ For example:
 
 You can also refer to [build
 guide](https://github.com/mzhaom/lrte/wiki/Build-Guide) if you prefer
-building everything from source code for further customization.
+building everything from source code for further customization.
+
+
+From a GCE machine, using a Debian VM
+=====================================
+```
+
+
+```
diff --git a/build_crosstool.sh b/build_crosstool.sh
@@ -39,6 +39,9 @@ CROSSTOOL_GCC_VERSION="4.9"
 # http://clang.llvm.org/get_started.html
 CROSSTOOL_CLANG_VERSION="4.0"
 
+CROSSTOOL_ISPC_VERSION="1.9.2dev"
+ISPC_GIT_COMMIT="d4a8afd"
+
 : ${crosstool_rpmver:="1.0"}
 # Update this each time new RPM's are built.
 : ${crosstool_rpmrel:="8"}
@@ -59,8 +62,9 @@ ln -sf /bin/bash /bin/sh
 
 # install packages that are needed by building binutils and clang
 # ncurses is required by cmake 3.6 even if I don't want it linked into llvm (since it is not crosscompiled)
+apt-get clean
 apt-get update --fix-missing
-apt-get install --fix-missing -y flex bison rpm texinfo texi2html libxml2-dev make alien wget python python-dev subversion libedit-dev libncurses5-dev swig
+apt-get install --fix-missing -y -q alien rpm
 
 function build_rpm() {
     local rpmrel=$1
@@ -94,6 +98,8 @@ function build_rpm() {
         --define "gcc_svn_version ${GCC_SVN_VERSION}" \
         --define "crosstool_clang_version ${CROSSTOOL_CLANG_VERSION}" \
         --define "clang_svn_version ${CLANG_SVN_VERSION}" \
+        --define "crosstool_ispc_version ${CROSSTOOL_ISPC_VERSION}" \
+        --define "ispc_git_commit ${ISPC_GIT_COMMIT}" \
         -bb ${spec_file}
 }
 
@@ -119,25 +125,40 @@ set -e
 # package foo ${CROSSTOOL_GCC_VERSION} ${GCC_SVN_VERSION}
 
 [ -z "${SKIP_CROSSTOOL_GCC}" ] && {
+    DEBIAN_FRONTEND=noninteractive apt-get install -y -q --fix-missing -y flex bison texinfo texi2html libxml2-dev make wget python python-dev subversion libedit-dev libncurses5-dev swig
     package gcc ${CROSSTOOL_GCC_VERSION} ${GCC_SVN_VERSION}
 }
 
-# Install the crosstool gcc to build clang
-dpkg -i ${DEB_DIR}/${GRTEBASENAME}-crosstool${CROSSTOOL_VERSION}-gcc-${CROSSTOOL_GCC_VERSION}_${crosstool_rpmver}-${crosstool_rpmrel}.${GCC_SVN_VERSION}svn_amd64.deb
-
-# Build cmake because cmake in ubuntu 13 is too old
-mkdir -p ${STAGING}/cmake
-CMAKE_VERSION=3.6.1
-if [ ! -e ${CROSSTOOL_SOURCES}/cmake-${CMAKE_VERSION}.tar.gz ]; then
-    pushd ${CROSSTOOL_SOURCES}
-    wget http://cmake.org/files/v3.6/cmake-${CMAKE_VERSION}.tar.gz
+[ -z "${SKIP_CROSSTOOL_CLANG}" ] && {
+    DEBIAN_FRONTEND=noninteractive apt-get install -y -q --fix-missing -y flex bison texinfo texi2html libxml2-dev make wget python python-dev subversion libedit-dev libncurses5-dev swig
+
+    # Install the crosstool gcc to build clang
+    dpkg -i ${DEB_DIR}/${GRTEBASENAME}-crosstool${CROSSTOOL_VERSION}-gcc-${CROSSTOOL_GCC_VERSION}_${crosstool_rpmver}-${crosstool_rpmrel}.${GCC_SVN_VERSION}svn_amd64.deb
+
+    # Build cmake because cmake in ubuntu 13 is too old
+    mkdir -p ${STAGING}/cmake
+    CMAKE_VERSION=3.6.1
+    if [ ! -e ${CROSSTOOL_SOURCES}/cmake-${CMAKE_VERSION}.tar.gz ]; then
+	pushd ${CROSSTOOL_SOURCES}
+	wget http://cmake.org/files/v3.6/cmake-${CMAKE_VERSION}.tar.gz
+	popd
+    fi
+    tar zxf ${CROSSTOOL_SOURCES}/cmake-${CMAKE_VERSION}.tar.gz -C ${STAGING}/cmake
+    pushd ${STAGING}/cmake/cmake-${CMAKE_VERSION}
+    ./configure --parallel=${PARALLELMFLAGS}
+    make ${PARALLELMFLAGS}
+    make install
     popd
-fi
-tar zxf ${CROSSTOOL_SOURCES}/cmake-${CMAKE_VERSION}.tar.gz -C ${STAGING}/cmake
-pushd ${STAGING}/cmake/cmake-${CMAKE_VERSION}
-./configure --parallel=${PARALLELMFLAGS}
-make ${PARALLELMFLAGS}
-make install
-popd
-
-package clang ${CROSSTOOL_CLANG_VERSION} ${CLANG_SVN_VERSION}
+
+    package clang ${CROSSTOOL_CLANG_VERSION} ${CLANG_SVN_VERSION}
+
+}
+
+[ -z "${SKIP_CROSSTOOL_ISPC}" ] && {
+    DEBIAN_FRONTEND=noninteractive apt-get install -y -q  flex bison make git python
+
+    dpkg -i ${DEB_DIR}/${GRTEBASENAME}-crosstool${CROSSTOOL_VERSION}-gcc-${CROSSTOOL_GCC_VERSION}_${crosstool_rpmver}-${crosstool_rpmrel}.${GCC_SVN_VERSION}svn_amd64.deb
+    dpkg -i ${DEB_DIR}/${GRTEBASENAME}-crosstool${CROSSTOOL_VERSION}-clang-${CROSSTOOL_CLANG_VERSION}_${crosstool_rpmver}-${crosstool_rpmrel}.${CLANG_SVN_VERSION}svn_amd64.deb
+
+    package ispc ${CROSSTOOL_ISPC_VERSION} ${ISPC_GIT_COMMIT}
+}
diff --git a/crosstool/crosstool-grte-ispc.spec b/crosstool/crosstool-grte-ispc.spec
@@ -0,0 +1,71 @@
+%define gcc_glibc_version gcc-%{grte_gcc_version}-%{grte_basename}
+%define base_summary Intel SPMD Program Compiler %{crosstool_ispc_version} for %{grte_root} with %{gcc_glibc_version}
+Summary: %{base_summary}
+Name: %{grte_basename}-crosstool%{crosstool_version}-ispc-%{crosstool_ispc_version}
+Version: %{crosstool_rpmver}
+Release: %{crosstool_rpmrel}
+License:  BSD 3-Clause from Intel, University of Illinois/NCSA for LLVM, GEORGIA TECH RESEARCH CORPORATION for the PTX parser, BSD 2-Clause from James Tursa for half precision floating point code.
+Group: Development/Languages
+Packager: Release Engineer <%{maintainer_email}>
+AutoReqProv: no
+
+BuildRoot: %{_tmppath}/%{name}-buildroot
+
+# Create debuginfo unless disable_debuginfo is defined.
+%define make_debuginfo %{?disable_debuginfo:""}%{!?disable_debuginfo:"t"}
+%define debug_cflags %{?disable_debuginfo:""}%{!?disable_debuginfo:"-g"}
+
+# Remove BuildRoot in %clean unless keep_buildroot is defined.
+%define remove_buildroot %{?keep_buildroot:""}%{!?keep_buildroot:"t"}
+
+# Enable bootstrapping unless disable_bootstrap is defined.
+%define bootstrap_config_arg %{?disable_bootstrap:"--disable-bootstrap"}%{!?disable_bootstrap:""}
+
+%description
+Intel SPMD Program Compiler (ISPC) %{crosstool_ispc_version} to work with %{grte_basename}, git commit @commit %{ispc_git_commit}
+
+
+%prep
+
+%define crosstool_top /usr/crosstool/%{crosstool_version}
+%define target_top %{crosstool_top}/%{gcc_glibc_version}
+%define grte_top %{grte_root}
+
+%build
+export PATH="%{grte_top}/bin:%{grte_top}/sbin:$PATH"
+
+rm -rf $RPM_BUILD_DIR
+WORK_DIR="$RPM_BUILD_DIR/$RPM_PACKAGE_NAME-$RPM_PACKAGE_VERSION"
+
+# TARGET=x86_64-unknown-linux-gnu
+# PREFIX="$RPM_BUILD_ROOT%{target_top}/x86"
+# Make gold the default linker
+mkdir -p ${WORK_DIR}/ispc-build
+cp -r %{_sourcedir}/ispc ${WORK_DIR}/ispc-build
+cd ${WORK_DIR}/ispc-build/ispc
+
+git apply %{_sourcedir}/ispc-d4a8afd.patch
+pwd
+ls
+export LLVM_HOME=/usr/crosstool/v2/%{gcc_glibc_version}/x86
+export PATH=${PATH}:${LLVM_HOME}/bin
+
+LLVM_HOME=${LLVM_HOME} PATH=${PATH}:${LLVM_HOME}/bin make
+
+%install
+
+WORK_DIR="$RPM_BUILD_DIR/$RPM_PACKAGE_NAME-$RPM_PACKAGE_VERSION"
+PREFIX="$RPM_BUILD_ROOT%{target_top}/x86"
+
+mkdir -p ${PREFIX}/bin
+cp ${WORK_DIR}/ispc-build/ispc/ispc ${PREFIX}/bin/ispc
+
+%clean
+
+%files
+%defattr(-,root,root)
+%{target_top}/x86/bin/ispc
+
+%changelog
+* Mon Oct 17 2016 Release Engineer <%{maintainer_email}>
+- ISPC ${crosstool_ispc_version} from git commit  %{ispc_git_commit}
diff --git a/crosstool/get-ispc-from-github.sh b/crosstool/get-ispc-from-github.sh
@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+
+# get the proper ispc commit
+
+COMMIT="d4a8afd6e8fe04969a26f69bca7e0c4d6f3ecd99"
+GIT_TOP=${GIT_TOP:-upstream}
+
+cd ${GIT_TOP}
+git clone https://github.com/ispc/ispc.git ispc
+cd ispc
+git reset --hard ${COMMIT}
+
+# patches?
diff --git a/release.py b/release.py
@@ -7,6 +7,7 @@
 import os
 import os.path
 import subprocess
+import shlex
 
 def start_container(docker_image, command,
                     container_name = None,
@@ -69,7 +70,7 @@ def start_container(docker_image, command,
         cmd.append(command)
     else:
         cmd += command
-    print("=== Running %s" % (str(cmd)))
+    print('+ '.join(shlex.quote(x) for x in cmd))
     if start_subprocess:
         subprocess.check_call(cmd)
     else:
@@ -120,7 +121,7 @@ def main():
                         help='Directory that stores original downloaded packages, like glibc code')
     parser.add_argument('--no_build_crosstool', action='store_true',
                         help='Skip building crosstool')
-    parser.add_argument('--crosstool_skip', choices=['gcc'], action='append',
+    parser.add_argument('--crosstool_skip', nargs='+', choices=['gcc','clang', 'ispc'], action='append',
                         help='Steps to skip when building crosstool')
 
     args = parser.parse_args()
@@ -186,7 +187,7 @@ def main():
             os.path.join(args.upstream_source, 'llvm/tools/clang'),
             'http://clang.llvm.org/get_started.html')
         if args.crosstool_skip:
-            for skip in args.crosstool_skip:
+            for skip in args.crosstool_skip[0]:
                 env['SKIP_CROSSTOOL_' + skip.upper()] = '1'
         start_container(args.docker_image,
                         ['./build_crosstool.sh', args.lrte_prefix,