use main

jenkins/Dockerfile/centos7_golang-1.13_rust

@@ -5,14 +5,21 @@ MAINTAINER Liu Yin <[email protected]>
 USER root
 WORKDIR /root
 
-RUN curl -L http://172.16.30.25/download/tools/cmake-3.10.0.tar.gz -o cmake-3.10.0.tar.gz \
-	&& tar xzf cmake-3.10.0.tar.gz \
-	&& cd cmake-3.10.0 \
-	&& ./bootstrap \
-	&& make \
-	&& make install \
+RUN yum install --nogpgcheck -y centos-release-scl \
+	&& yum install -y devtoolset-8 \
+	&& yum clean all
+
+RUN yum remove -y cmake \
+	&& curl -L https://github.com/Kitware/CMake/releases/download/v3.10.0/cmake-3.10.0-linux-x86_64.sh -o cmake-3.10.0.sh \
+	&& sh cmake-3.10.0.sh --skip-license --prefix=/usr/local \
 	&& cmake --version \
-	&& rm -rf /cmake-3.10.0.tar.gz /cmake-3.10.0
+	&& rm -rf cmake-3.10.0.sh
+
+# This is a workaround, we need to figure out why there is no jq in this image
+RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+RUN yum makecache \
+       && yum update -y \
+       && yum install --nogpgcheck -y jq
 
 RUN curl -L https://github.com/gflags/gflags/archive/v2.1.2.tar.gz -o gflags.tar.gz \
 	&& tar xf gflags.tar.gz \

jenkins/Dockerfile/release/br-arm64

@@ -0,0 +1,3 @@
+FROM centos:8
+COPY zoneinfo.zip  /usr/local/go/lib/time/zoneinfo.zip
+COPY br /br

jenkins/Dockerfile/release/dumpling-arm64

@@ -0,0 +1,3 @@
+FROM centos:8
+COPY zoneinfo.zip  /usr/local/go/lib/time/zoneinfo.zip
+COPY dumpling /dumpling

jenkins/Dockerfile/release/pd-arm64

@@ -0,0 +1,11 @@
+FROM centos:8
+COPY pd-server /pd-server
+COPY pd-ctl /pd-ctl
+COPY pd-recover /pd-recover
+RUN set -e && \
+    curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-8.repo && \
+    sed -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo && \
+    dnf install bind-utils wget -y && \
+    dnf clean all
+EXPOSE 2379 2380
+ENTRYPOINT ["/pd-server"]

jenkins/Dockerfile/release/ticdc-arm64

@@ -0,0 +1,5 @@
+FROM alpine:3.12
+RUN apk add --no-cache tzdata bash curl socat
+COPY cdc /cdc
+EXPOSE 8300
+CMD [ "/cdc" ]

jenkins/Dockerfile/release/tidb-arm64

@@ -0,0 +1,5 @@
+FROM centos:8
+RUN yum install -y curl
+COPY tidb-server /tidb-server
+EXPOSE 4000
+ENTRYPOINT ["/tidb-server"]

jenkins/Dockerfile/release/tidb-binlog-arm64

@@ -0,0 +1,9 @@
+FROM centos:8
+COPY zoneinfo.zip  /usr/local/go/lib/time/zoneinfo.zip
+COPY pump /pump
+COPY drainer /drainer
+COPY reparo /reparo
+COPY binlogctl /binlogctl
+EXPOSE 4000
+EXPOSE 8249 8250
+CMD ["/pump"]

jenkins/Dockerfile/release/tidb-lightning-arm64

@@ -0,0 +1,6 @@
+FROM centos:8
+COPY zoneinfo.zip  /usr/local/go/lib/time/zoneinfo.zip
+COPY tidb-lightning /tidb-lightning
+COPY tidb-lightning-ctl /tidb-lightning-ctl
+COPY tikv-importer /tikv-importer
+COPY br /br

jenkins/Dockerfile/release/tidb-monitor-reloader-arm64

@@ -0,0 +1,4 @@
+FROM centos:8
+COPY ./build/linux/reload  /bin/reload
+ENTRYPOINT ["/bin/reload"]
+CMD        [ "--watch-path=/etc/prometheus/rules", "--prometheus-url=http://127.0.0.1:9090" ]

jenkins/Dockerfile/release/tiflash-arm64

@@ -0,0 +1,9 @@
+FROM hub.pingcap.net/jenkins/centos7_arm64:cached
+USER root
+WORKDIR /root/
+ENV HOME /root/
+ENV TZ Asia/Shanghai
+ENV LD_LIBRARY_PATH /tiflash
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+COPY tiflash /tiflash
+ENTRYPOINT ["/tiflash/tiflash", "server"]

jenkins/Dockerfile/release/tikv-arm64

@@ -0,0 +1,7 @@
+FROM centos:8
+ENV TZ=/etc/localtime
+ENV TZDIR=/usr/share/zoneinfo
+COPY tikv-server /tikv-server
+COPY tikv-ctl /tikv-ctl
+EXPOSE 20160
+ENTRYPOINT ["/tikv-server"]

jenkins/pipelines/build-linux-amd64-enterprise.groovy

@@ -0,0 +1,232 @@
+/*
+* @RELEASE_TAG
+*/
+def slackcolor = 'good'
+def githash
+def os = "linux"
+def arch = "amd64"
+def TIDB_HASH
+def TIKV_HASH
+def PD_HASH
+def TIFLASH_HASH
+
+def label = "build-tiflash-release"
+
+try {
+    node("build_go1130") {
+        container("golang") {
+            def ws = pwd()
+            deleteDir()
+
+            stage("GO node") {
+                println "debug command:\nkubectl -n jenkins-ci exec -ti ${NODE_NAME} bash"
+                println "${ws}"
+                sh "curl -s ${FILE_SERVER_URL}/download/builds/pingcap/ee/gethash.py > gethash.py"
+                TIDB_HASH = sh(returnStdout: true, script: "python gethash.py -repo=tidb -version=${RELEASE_TAG} -s=${FILE_SERVER_URL}").trim()
+                TIKV_HASH = sh(returnStdout: true, script: "python gethash.py -repo=tikv -version=${RELEASE_TAG} -s=${FILE_SERVER_URL}").trim()
+                PD_HASH = sh(returnStdout: true, script: "python gethash.py -repo=pd -version=${RELEASE_TAG} -s=${FILE_SERVER_URL}").trim()
+            }
+
+            stage("Build tidb") {
+                dir("go/src/github.com/pingcap/tidb") {
+                    if (sh(returnStatus: true, script: '[ -d .git ] || git rev-parse --git-dir > /dev/null 2>&1') != 0) {
+                        deleteDir()
+                    }
+
+                    def target = "tidb-server"
+                    // def refspath = "refs/pingcap/tidb/${RELEASE_TAG}/sha1"
+                    def filepath = "builds/pingcap/tidb/${TIDB_HASH}/centos7/tidb-server-${RELEASE_TAG}-enterprise.tar.gz"
+
+                    // checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name:  "${TIDB_HASH}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: '+refs/heads/*:refs/remotes/origin/*', url: '[email protected]:pingcap/tidb.git']]]
+                    if(RELEASE_TAG == "master") {
+                        checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name:  "${TIDB_HASH}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: '+refs/heads/*:refs/remotes/origin/*', url: '[email protected]:pingcap/tidb.git']]]
+                    } else {
+                        checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name:  "${RELEASE_TAG}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'LocalBranch'],[$class: 'CloneOption', noTags: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: "+refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}", url: '[email protected]:pingcap/tidb.git']]]
+                    }
+
+                    sh """
+                    mkdir -p ${ws}/go/pkg && ln -sf \$GOPATH/pkg/mod ${ws}/go/pkg/mod
+                    make clean
+                    go version
+                    GOPATH=${ws}/go TIDB_EDITION=Enterprise make
+                    tar --exclude=${target}.tar.gz -czvf ${target}.tar.gz *
+                    curl -F ${filepath}=@${target}.tar.gz ${FILE_SERVER_URL}/upload
+                    """
+                }
+            }
+            stage("Build plugins") {
+                dir("go/src/github.com/pingcap/tidb-build-plugin") {
+                    deleteDir()
+                    timeout(20) {
+                        sh """
+                       cp -R ${ws}/go/src/github.com/pingcap/tidb/. ./
+                       cd cmd/pluginpkg
+                       go build
+                       """
+                    }
+                }
+                dir("go/src/github.com/pingcap/enterprise-plugin") {
+                    checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name: "release-4.0"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', timeout: 2]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: '+refs/heads/*:refs/remotes/origin/*', url: '[email protected]:pingcap/enterprise-plugin.git']]]
+                    def plugin_hash = sh(returnStdout: true, script: "git rev-parse HEAD").trim()
+                }
+                def filepath_whitelist = "builds/pingcap/tidb-plugins/optimization/enterprise/${RELEASE_TAG}/centos7/whitelist-1.so"
+                def md5path_whitelist = "builds/pingcap/tidb-plugins/optimization/enterprise/${RELEASE_TAG}/centos7/whitelist-1.so.md5"
+                def filepath_audit = "builds/pingcap/tidb-plugins/optimization/enterprise/${RELEASE_TAG}/centos7/audit-1.so"
+                def md5path_audit = "builds/pingcap/tidb-plugins/optimization/enterprise/${RELEASE_TAG}/centos7/audit-1.so.md5"
+                dir("go/src/github.com/pingcap/enterprise-plugin/whitelist") {
+                    sh """
+                   GOPATH=${ws}/go ${ws}/go/src/github.com/pingcap/tidb-build-plugin/cmd/pluginpkg/pluginpkg -pkg-dir ${ws}/go/src/github.com/pingcap/enterprise-plugin/whitelist -out-dir ${ws}/go/src/github.com/pingcap/enterprise-plugin/whitelist
+                   """
+                    sh """
+                            md5sum whitelist-1.so > whitelist-1.so.md5
+                            curl -F ${md5path_whitelist}[email protected] ${FILE_SERVER_URL}/upload
+                            curl -F ${filepath_whitelist}[email protected] ${FILE_SERVER_URL}/upload
+                            """
+                }
+                dir("go/src/github.com/pingcap/enterprise-plugin/audit") {
+                    sh """
+                   GOPATH=${ws}/go ${ws}/go/src/github.com/pingcap/tidb-build-plugin/cmd/pluginpkg/pluginpkg -pkg-dir ${ws}/go/src/github.com/pingcap/enterprise-plugin/audit -out-dir ${ws}/go/src/github.com/pingcap/enterprise-plugin/audit
+                   """
+                    sh """
+                        md5sum audit-1.so > audit-1.so.md5
+                        curl -F ${md5path_audit}[email protected] ${FILE_SERVER_URL}/upload
+                        curl -F ${filepath_audit}[email protected] ${FILE_SERVER_URL}/upload
+                        """
+                }
+            }
+
+            stage("Build pd") {
+                dir("go/src/github.com/pingcap/pd") {
+                    if (sh(returnStatus: true, script: '[ -d .git ] || git rev-parse --git-dir > /dev/null 2>&1') != 0) {
+                        deleteDir()
+                    }
+
+                    def target = "pd-server"
+                    // def refspath = "refs/pingcap/pd/${RELEASE_TAG}/sha1"
+                    def filepath = "builds/pingcap/pd/${PD_HASH}/centos7/pd-server-${RELEASE_TAG}-enterprise.tar.gz"
+
+                    // checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name: "${PD_HASH}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: '+refs/heads/*:refs/remotes/origin/*', url: '[email protected]:pingcap/pd.git']]]
+                    if(RELEASE_TAG == "master") {
+                        checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name: "${PD_HASH}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: '+refs/heads/*:refs/remotes/origin/*', url: '[email protected]:tikv/pd.git']]]
+                    } else {
+                        checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name:  "${RELEASE_TAG}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'LocalBranch'],[$class: 'CloneOption', noTags: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: "+refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}", url: '[email protected]:tikv/pd.git']]]
+                    }
+
+                    sh """
+                    go version
+                    PD_EDITION=Enterprise make
+                    PD_EDITION=Enterprise make tools
+                    tar --exclude=${target}.tar.gz -czvf ${target}.tar.gz *
+                    curl -F ${filepath}=@${target}.tar.gz ${FILE_SERVER_URL}/upload
+                    """
+                }
+            }
+        }
+    }
+
+    node("build") {
+        container("rust") {
+            stage("Rust node") {
+                println "debug command:\nkubectl -n jenkins-ci exec -ti ${NODE_NAME} bash"
+            }
+
+            stage("Build TiKV") {
+                dir("go/src/github.com/pingcap/tikv") {
+                    if (sh(returnStatus: true, script: '[ -d .git ] || git rev-parse --git-dir > /dev/null 2>&1') != 0) {
+                        deleteDir()
+                    }
+
+                    def target = "tikv-server"
+                    // def refspath = "refs/pingcap/tikv/${RELEASE_TAG}/sha1"
+                    def filepath = "builds/pingcap/tikv/${TIKV_HASH}/centos7/tikv-server-${RELEASE_TAG}-enterprise.tar.gz"
+
+                    // checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name: "${TIKV_HASH}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: '+refs/heads/*:refs/remotes/origin/*', url: '[email protected]:tikv/tikv.git']]]
+                    if(RELEASE_TAG == "master") {
+                        checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name: "${TIKV_HASH}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: '+refs/heads/*:refs/remotes/origin/*', url: '[email protected]:tikv/tikv.git']]]
+                    } else {
+                        checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name:  "${RELEASE_TAG}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'LocalBranch'],[$class: 'CloneOption', noTags: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: "+refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}", url: '[email protected]:tikv/tikv.git']]]
+                    }
+
+                    sh """
+                    grpcio_ver=`grep -A 1 'name = "grpcio"' Cargo.lock | tail -n 1 | cut -d '"' -f 2`
+                    if [[ ! "0.8.0" > "\$grpcio_ver" ]]; then
+                        echo using gcc 8
+                        source /opt/rh/devtoolset-8/enable
+                    fi
+                    TIKV_EDITION=Enterprise CARGO_TARGET_DIR=.target ROCKSDB_SYS_STATIC=1 ROCKSDB_SYS_SSE=0 make dist_release
+                    tar --exclude=${target}.tar.gz -czvf ${target}.tar.gz *
+                    curl -F ${filepath}=@${target}.tar.gz ${FILE_SERVER_URL}/upload
+                    """
+                }
+            }
+        }
+    }
+
+    podTemplate(name: label, label: label,
+            nodeSelector: 'role_type=slave',
+            instanceCap: 5, idleMinutes: 10,
+            workspaceVolume: emptyDirWorkspaceVolume(memory: true),
+            containers: [
+                    containerTemplate(name: 'dockerd', image: 'docker:18.09.6-dind', privileged: true),
+                    containerTemplate(name: 'docker', image: 'hub.pingcap.net/zyguan/docker:build-essential',
+                            alwaysPullImage: false, envVars: [
+                            envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2375'),
+                    ], ttyEnabled: true, command: 'cat'),
+                    containerTemplate(name: 'builder', image: 'hub.pingcap.net/tiflash/tiflash-builder',
+                            alwaysPullImage: true, ttyEnabled: true, privileged: true, command: 'cat',
+                            resourceRequestCpu: '12000m', resourceRequestMemory: '20Gi',
+                            resourceLimitCpu: '16000m', resourceLimitMemory: '48Gi'),
+            ]) {
+        node(label) {
+            container("builder") {
+                stage("Build tiflash - prepare") {
+                    deleteDir()
+                    sh "rm -rf ./*"
+                    println "debug command:\nkubectl -n jenkins-ci exec -ti ${NODE_NAME} bash"
+
+                    sh "curl -s ${FILE_SERVER_URL}/download/builds/pingcap/ee/gethash.py > gethash.py"
+                    TIFLASH_HASH = sh(returnStdout: true, script: "python gethash.py -repo=tics -version=${RELEASE_TAG} -s=${FILE_SERVER_URL}").trim()
+
+                }
+                stage("Build tiflash - build") {
+                    dir("tics") {
+                        def target = "tiflash"
+                        def filepath
+
+                        filepath = "builds/pingcap/tiflash/${RELEASE_TAG}/${TIFLASH_HASH}/centos7/tiflash-${RELEASE_TAG}-enterprise.tar.gz"
+
+                        retry(10) {
+                            if(RELEASE_TAG == "master") {
+                                checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name: "${TIFLASH_HASH}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CheckoutOption', timeout: 30], [$class: 'CloneOption', timeout: 60], [$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout'], [$class: 'SubmoduleOption', disableSubmodules: false, parentCredentials: true, recursiveSubmodules: true, trackingSubmodules: false, reference: '', shallow: true, threads: 8], [$class: 'LocalBranch']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: '+refs/heads/*:refs/remotes/origin/*', url: '[email protected]:pingcap/tics.git']]]
+                            } else {
+                                checkout changelog: false, poll: true, scm: [$class: 'GitSCM', branches: [[name: "${RELEASE_TAG}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CheckoutOption', timeout: 30], [$class: 'CloneOption', timeout: 60], [$class: 'LocalBranch'], [$class: 'CloneOption', noTags: true], [$class: 'SubmoduleOption', disableSubmodules: false, parentCredentials: true, recursiveSubmodules: true, trackingSubmodules: false, reference: '', shallow: true, threads: 8]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-sre-bot-ssh', refspec: "+refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}", url: '[email protected]:pingcap/tics.git']]]
+                            }
+                        }
+
+                        sh """
+                        TIFLASH_EDITION=Enterprise NPROC=12 release-centos7/build/build-release.sh
+                        cd release-centos7/
+                        tar -czvf tiflash.tar.gz tiflash
+                        curl -F ${filepath}=@${target}.tar.gz ${FILE_SERVER_URL}/upload
+                        """
+                    }
+                }
+            }
+        }
+
+    }
+
+    currentBuild.result = "SUCCESS"
+} catch (Exception e) {
+    currentBuild.result = "FAILURE"
+    slackcolor = 'danger'
+    echo "${e}"
+}
+
+stage('Summary') {
+    echo "Send slack here ..."
+    //slackSend channel: "", color: "${slackcolor}", teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}"
+    // if (currentBuild.result != "SUCCESS") {
+    //     slackSend channel: '#jenkins-ci-build-critical', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}"
+    // }
+}