diff --git a/jenkins/jobs/ci2/pingcap/tispark/ghpr_integration_test.groovy b/jenkins/jobs/ci2/pingcap/tispark/ghpr_integration_test.groovy deleted file mode 100644 index 3866200a98..0000000000 --- a/jenkins/jobs/ci2/pingcap/tispark/ghpr_integration_test.groovy +++ /dev/null @@ -1,79 +0,0 @@ -// REF: https:///plugin/job-dsl/api-viewer/index.html -pipelineJob('tispark_ghpr_integration_test') { - logRotator { - daysToKeep(30) - } - parameters { - stringParam("ghprbActualCommit") - stringParam("ghprbPullId") - } - properties { - // priority(0) // 0 fast than 1 - githubProjectUrl("https://github.com/pingcap/tispark/") - pipelineTriggers { - triggers { - ghprbTrigger { - cron('H/5 * * * *') - gitHubAuthId('37c47302-ce04-4cae-a76f-b75f439c1464') - - triggerPhrase('.*/run(-all-tests|-integration-tests|-integration-common-test).*') - onlyTriggerPhrase(true) - skipBuildPhrase(".*skip-ci.*") - buildDescTemplate('PR #$pullId: $abbrTitle\n$url') - whitelist("") - orgslist("pingcap") - // ignore when only those file changed.( - // multi line regex - // excludedRegions('.*\\.md') - excludedRegions('') // current the context is required in github branch protection. - - blackListLabels("") - whiteListLabels("") - adminlist("") - blackListCommitAuthor("") - includedRegions("") - commentFilePath("") - - allowMembersOfWhitelistedOrgsAsAdmin(true) - permitAll(true) - useGitHubHooks(true) - displayBuildErrorsOnDownstreamBuilds(false) - autoCloseFailedPullRequests(false) - - // useless, but can not delete. - commitStatusContext("--none--") - msgSuccess("--none--") - msgFailure("--none--") - - extensions { - ghprbCancelBuildsOnUpdate { overrideGlobal(true) } - ghprbSimpleStatus { - commitStatusContext("idc-jenkins-ci-tispark/integration-test") - statusUrl('${RUN_DISPLAY_URL}') - startedStatus("Jenkins job is running.") - triggeredStatus("Jenkins job triggered.") - addTestResults(false) - showMatrixStatus(false) - } - } - } - } - } - } - - definition { - cpsScm { - lightweight(true) - scriptPath("jenkins/pipelines/ci/tispark/integration_test.groovy") - scm { - git{ - remote { - url('git@github.com:PingCAP-QE/ci.git') - credentials('github-sre-bot-ssh') - } - branch('main') - } - } - } - } -} diff --git a/jenkins/pipelines/ci/tispark/integration_test.groovy b/jenkins/pipelines/ci/tispark/integration_test.groovy deleted file mode 100644 index f8ee4d0150..0000000000 --- a/jenkins/pipelines/ci/tispark/integration_test.groovy +++ /dev/null @@ -1,390 +0,0 @@ - -credentialsId= "github-sre-bot-ssh" - -env.GOROOT = "/usr/local/go" -env.GOPATH = "/go" -env.PATH = "/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin" -env.PATH = "${env.GOROOT}/bin:/home/jenkins/bin:/bin:${env.PATH}" -def TIDB_BRANCH = "master" -def TIKV_BRANCH = "master" -def PD_BRANCH = "master" -def TIFLASH_BRANCH = "master" -def MVN_PROFILE = "-Pjenkins" -def TEST_MODE = "full" -def PARALLEL_NUMBER = 18 -def TEST_REGION_SIZE = "normal" -def TEST_TIFLASH = "false" -def TEST_ALTER_PRIMARY_KEY = "true" -def TEST_SPARK_CATALOG = "true" - -// parse tidb branch -def m1 = ghprbCommentBody =~ /tidb\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m1) { - TIDB_BRANCH = "${m1[0][1]}" -} -m1 = null -println "TIDB_BRANCH=${TIDB_BRANCH}" - -// parse pd branch -def m2 = ghprbCommentBody =~ /pd\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m2) { - PD_BRANCH = "${m2[0][1]}" -} -m2 = null -println "PD_BRANCH=${PD_BRANCH}" - -// parse tikv branch -def m3 = ghprbCommentBody =~ /tikv\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m3) { - TIKV_BRANCH = "${m3[0][1]}" -} -m3 = null -println "TIKV_BRANCH=${TIKV_BRANCH}" - -// parse tiflash branch -def m4 = ghprbCommentBody =~ /tiflash\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m4) { - TIFLASH_BRANCH = "${m4[0][1]}" -} -m4 = null -println "TIFLASH_BRANCH=${TIFLASH_BRANCH}" - -// parse mvn profile -def m5 = ghprbCommentBody =~ /profile\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m5) { - for (int i = 0 ; i < m5.getCount(); i++) { - MVN_PROFILE = MVN_PROFILE + " -P${m5[i][1]}" - } -} -m5 = null - -// parse test mode -def m6 = ghprbCommentBody =~ /mode\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m6) { - TEST_MODE = "${m6[0][1]}" -} -m6 = null - -// parse test region size -def m7 = ghprbCommentBody =~ /region\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m7) { - TEST_REGION_SIZE = "${m7[0][1]}" -} -m7 = null - -// parse test tiflash -def m8 = ghprbCommentBody =~ /test-flash\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m8) { - TEST_TIFLASH = "${m8[0][1]}" -} -m8 = null - -// parse test alter primary key -def m9 = ghprbCommentBody =~ /test-alter-primary-key\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m9) { - TEST_ALTER_PRIMARY_KEY = "${m9[0][1]}" -} -m9 = null - -// parse test spark catalog -def m10 = ghprbCommentBody =~ /test-spark-catalog\s*=\s*([^\s\\]+)(\s|\\|$)/ -if (m10) { - TEST_SPARK_CATALOG = "${m10[0][1]}" -} -m10 = null - -groovy.lang.Closure readfile = { filename -> - def file = readFile filename - return file.split("\n") as List -} - -groovy.lang.Closure get_mvn_str = { total_chunks -> - def mvnStr = " -DwildcardSuites=" - for (int i = 0 ; i < total_chunks.size() - 1; i++) { - // print total_chunks - def trimStr = total_chunks[i] - mvnStr = mvnStr + "${trimStr}," - } - def trimStr = total_chunks[total_chunks.size() - 1] - mvnStr = mvnStr + "${trimStr}" - mvnStr = mvnStr + " -DfailIfNoTests=false" - mvnStr = mvnStr + " -DskipAfterFailureCount=1" - return mvnStr -} - -def label = "regression-test-tispark-${BUILD_NUMBER}" - -podTemplate(name: label, label: label, cloud: "kubernetes-ksyun", instanceCap: 12, namespace: 'jenkins-tispark', nodeSelector: "kubernetes.io/arch=amd64", - containers: [ - containerTemplate(name: 'golang', image: 'hub.pingcap.net/jenkins/centos7_golang-1.12:cached', - envVars: [ - envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2375'), - ], alwaysPullImage: true, ttyEnabled: true, command: 'cat'), - containerTemplate(name: 'java', image: 'hub.pingcap.net/jenkins/centos7_golang-1.13_java:cached', - resourceRequestCpu: '8000m', - resourceRequestMemory: '24Gi', - envVars: [ - envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2375'), - ], alwaysPullImage: true, ttyEnabled: true, command: 'cat'), -]) { - catchError { - stage('Prepare') { - node (label) { - println "${NODE_NAME}" - container("golang") { - deleteDir() - - // tidb - def tidb_sha1 = sh(returnStdout: true, script: "curl -f ${FILE_SERVER_URL}/download/refs/pingcap/tidb/${TIDB_BRANCH}/sha1").trim() - sh "curl -C - --retry 3 -f ${FILE_SERVER_URL}/download/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz | tar xz" - // tikv - def tikv_sha1 = sh(returnStdout: true, script: "curl -f ${FILE_SERVER_URL}/download/refs/pingcap/tikv/${TIKV_BRANCH}/sha1").trim() - sh "curl -C - --retry 3 -f ${FILE_SERVER_URL}/download/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz | tar xz" - // pd - def pd_sha1 = sh(returnStdout: true, script: "curl -f ${FILE_SERVER_URL}/download/refs/pingcap/pd/${PD_BRANCH}/sha1").trim() - sh "curl -C - --retry 3 -f ${FILE_SERVER_URL}/download/builds/pingcap/pd/${pd_sha1}/centos7/pd-server.tar.gz | tar xz" - // tiflash - if (TEST_TIFLASH != "false") { - def tiflash_sha1 = sh(returnStdout: true, script: "curl -f ${FILE_SERVER_URL}/download/refs/pingcap/tiflash/${TIFLASH_BRANCH}/sha1").trim() - sh "curl -C - --retry 3 -f ${FILE_SERVER_URL}/download/builds/pingcap/tiflash/${TIFLASH_BRANCH}/${tiflash_sha1}/centos7/tiflash.tar.gz | tar xz" - stash includes: "tiflash/**", name: "tiflash_binary" - } - // alter-primary-key - if (TEST_ALTER_PRIMARY_KEY == "false") { - sh "sed -i 's/alter-primary-key = true/alter-primary-key = false/' config/tidb.toml" - sh "sed -i 's/alter-primary-key = true/alter-primary-key = false/' config/tidb-4.0.toml" - } - - stash includes: "bin/**", name: "binaries" - - def specStr = "+refs/heads/*:refs/remotes/origin/*" - if (ghprbPullId != null && ghprbPullId != "") { - specStr = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" - } - - dir("/home/jenkins/agent/git/tispark") { - if (sh(returnStatus: true, script: '[ -d .git ] && [ -f Makefile ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) { - deleteDir() - } - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentialsId, refspec: specStr, url: 'git@github.com:pingcap/tispark.git']]] - } - - dir("go/src/github.com/pingcap/tispark") { - deleteDir() - sh """ - pwd - cp -R /home/jenkins/agent/git/tispark/. ./ - git checkout -f ${ghprbActualCommit} - find core/src -name '*Suite*' | grep -v 'MultiColumnPKDataTypeSuite\\|PartitionTableSuite' > test - shuf test -o test2 - mv test2 test - """ - - if (TEST_REGION_SIZE != "normal") { - sh "sed -i 's/\\# region-max-size = \\\"2MB\\\"/region-max-size = \\\"2MB\\\"/' config/tikv.toml" - sh "sed -i 's/\\# region-split-size = \\\"1MB\\\"/region-split-size = \\\"1MB\\\"/' config/tikv.toml" - sh "cat config/tikv.toml" - } - - if (TEST_MODE != "simple") { - sh """ - find core/src -name '*MultiColumnPKDataTypeSuite*' >> test - """ - } - - if (TEST_TIFLASH != "false") { - sh "wget https://raw.githubusercontent.com/PingCAP-QE/ci/main/jenkins/pipelines/ci/tispark/tidb_config-for-tiflash-test.properties" - sh "cp tidb_config-for-tiflash-test.properties core/src/test/resources/tidb_config.properties" - } - - if (TEST_SPARK_CATALOG != "false") { - sh """ - touch core/src/test/resources/tidb_config.properties - echo "spark.sql.catalog.tidb_catalog=org.apache.spark.sql.catalyst.catalog.TiCatalog" >> core/src/test/resources/tidb_config.properties - """ - } - - sh """ - sed -i 's/core\\/src\\/test\\/scala\\///g' test - sed -i 's/\\//\\./g' test - sed -i 's/\\.scala//g' test - split test -n r/$PARALLEL_NUMBER test_unit_ -a 2 --numeric-suffixes=1 - """ - - for (int i = 1; i <= PARALLEL_NUMBER; i++) { - if (i < 10) { - sh """cat test_unit_0$i""" - } else { - sh """cat test_unit_$i""" - } - } - - sh """ - wget https://raw.githubusercontent.com/PingCAP-QE/ci/main/jenkins/pipelines/ci/tispark/log4j-ci.properties - cp log4j-ci.properties core/src/test/resources/log4j.properties - bash core/scripts/version.sh - bash core/scripts/fetch-test-data.sh - mv core/src/test core-test/src/ - bash tikv-client/scripts/proto.sh - """ - } - - stash includes: "go/src/github.com/pingcap/tispark/**", name: "tispark", useDefaultExcludes: false - } - } - } - - stage('Integration Tests') { - timeout(150){ - def tests = [:] - - groovy.lang.Closure run_tispark_test = { chunk_suffix -> - dir("go/src/github.com/pingcap/tispark") { - if (chunk_suffix < 10) { - run_chunks = readfile("test_unit_0${chunk_suffix}") - } else { - run_chunks = readfile("test_unit_${chunk_suffix}") - } - - print run_chunks - def mvnStr = get_mvn_str(run_chunks) - sh """ - rm -rf /maven/.m2/repository/* - rm -rf /maven/.m2/settings.xml - rm -rf ~/.m2/settings.xml - archive_url=http://fileserver.pingcap.net/download/builds/pingcap/tispark/cache/tispark-m2-cache-latest.tar.gz - if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -C - --retry 3 -sfL \$archive_url | tar -zx -C /maven || true; fi - """ - sh """ - export MAVEN_OPTS="-Xmx6G -XX:MaxMetaspaceSize=1024M" - mvn clean package ${MVN_PROFILE} -DskipTests - mvn test ${MVN_PROFILE} -Dtest=moo ${mvnStr} - """ - } - } - - groovy.lang.Closure run_tikvclient_test = { chunk_suffix -> - withCredentials([string(credentialsId: 'codecov-token-tispark', variable: 'CODECOV_TOKEN')]) { - dir("go/src/github.com/pingcap/tispark") { - sh """ - rm -rf /maven/.m2/repository/* - rm -rf /maven/.m2/settings.xml - rm -rf ~/.m2/settings.xml - archive_url=http://fileserver.pingcap.net/download/builds/pingcap/tispark/cache/tispark-m2-cache-latest.tar.gz - if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -C - --retry 3 -sfL \$archive_url | tar -zx -C /maven || true; fi - """ - sh """ - export MAVEN_OPTS="-Xmx6G -XX:MaxMetaspaceSize=1024M" - mvn test ${MVN_PROFILE} -am -pl tikv-client - """ - sh 'curl -s https://codecov.io/bash | bash -s' - } - } - } - - groovy.lang.Closure run_intergration_test = { chunk_suffix, run_test -> - node(label) { - println "${NODE_NAME}" - container("java") { - deleteDir() - unstash 'binaries' - unstash 'tispark' - if (TEST_TIFLASH != "false") { - unstash 'tiflash_binary' - } - - try { - - groovy.lang.Closure isv4 = { branch_name -> - if (branch_name.startsWith("v4") || branch_name.startsWith("release-4")) { - return true - } - return false - } - - if (isv4(TIDB_BRANCH) || isv4(TIKV_BRANCH) || isv4(PD_BRANCH)) { - sh """ - rm go/src/github.com/pingcap/tispark/config/pd.toml - rm go/src/github.com/pingcap/tispark/config/tikv.toml - rm go/src/github.com/pingcap/tispark/config/tidb.toml - - mv go/src/github.com/pingcap/tispark/config/pd-4.0.toml go/src/github.com/pingcap/tispark/config/pd.toml - mv go/src/github.com/pingcap/tispark/config/tikv-4.0.toml go/src/github.com/pingcap/tispark/config/tikv.toml - mv go/src/github.com/pingcap/tispark/config/tidb-4.0.toml go/src/github.com/pingcap/tispark/config/tidb.toml - """ - } - - sh """ - #sudo sysctl -w net.ipv4.ip_local_port_range=\'1000 30000\' - killall -9 tidb-server || true - killall -9 tikv-server || true - killall -9 pd-server || true - killall -9 tiflash || true - killall -9 java || true - touch tiflash_cmd_line.log - sleep 10 - bin/pd-server --name=pd --data-dir=pd --config=go/src/github.com/pingcap/tispark/config/pd.toml &>pd.log & - sleep 10 - bin/tikv-server --pd=127.0.0.1:2379 -s tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 --config=go/src/github.com/pingcap/tispark/config/tikv.toml &>tikv.log & - sleep 10 - ps aux | grep '-server' || true - curl -s 127.0.0.1:2379/pd/api/v1/status || true - bin/tidb-server --store=tikv --path="127.0.0.1:2379" --config=go/src/github.com/pingcap/tispark/config/tidb.toml &>tidb.log & - sleep 60 - """ - - if (TEST_TIFLASH != "false") { - sh """ - export LD_LIBRARY_PATH=/home/jenkins/agent/workspace/tispark_ghpr_integration_test/tiflash - ls -l \$LD_LIBRARY_PATH - tiflash/tiflash server config --config-file go/src/github.com/pingcap/tispark/config/tiflash.toml &>tiflash_cmd_line.log & - ps aux | grep 'tiflash' - sleep 60 - """ - sh "ps aux | grep 'tiflash'" - } - - timeout(240) { - run_test(chunk_suffix) - } - } catch (err) { - sh """ - ps aux | grep '-server' || true - curl -s 127.0.0.1:2379/pd/api/v1/status || true - """ - sh "cat pd.log" - sh "cat tikv.log" - sh "cat tidb.log" - if (TEST_TIFLASH != "false") { - sh """ - touch tiflash_cmd_line.log - touch tiflash/tiflash_error.log - touch tiflash/tiflash.log - touch tiflash/tiflash_cluster_manager.log - touch tiflash/tiflash_tikv.log - """ - sh "cat tiflash_cmd_line.log" - sh "cat tiflash/tiflash_error.log" - sh "cat tiflash/tiflash.log" - sh "cat tiflash/tiflash_cluster_manager.log" - sh "cat tiflash/tiflash_tikv.log" - } - throw err - } - } - } - } - - for (int i = 1; i <= PARALLEL_NUMBER; i++) { - int x = i - tests["Integration test = $i"] = {run_intergration_test(x, run_tispark_test)} - } - tests["Integration tikv-client test"] = {run_intergration_test(0, run_tikvclient_test)} - - parallel tests - } - } - - currentBuild.result = "SUCCESS" - } -} diff --git a/jobs/pingcap/tiflash/latest/pull_integration_test.groovy b/jobs/pingcap/tiflash/latest/pull_integration_test.groovy index 1f07f1fa66..2526b00703 100644 --- a/jobs/pingcap/tiflash/latest/pull_integration_test.groovy +++ b/jobs/pingcap/tiflash/latest/pull_integration_test.groovy @@ -14,13 +14,13 @@ pipelineJob('pingcap/tiflash/pull_integration_test') { // priority(0) // 0 fast than 1 githubProjectUrl("https://github.com/pingcap/tiflash") } - + definition { cpsScm { lightweight(true) scriptPath("pipelines/pingcap/tiflash/latest/pull_integration_test.groovy") scm { - git{ + git { remote { url('https://github.com/PingCAP-QE/ci.git') } diff --git a/jobs/pingcap/tispark/aa_folder.groovy b/jobs/pingcap/tispark/aa_folder.groovy new file mode 100644 index 0000000000..49e5f5c095 --- /dev/null +++ b/jobs/pingcap/tispark/aa_folder.groovy @@ -0,0 +1,3 @@ +folder('pingcap/tispark') { + description("Folder for pipelines of pingcap/tispark repo") +} diff --git a/jobs/pingcap/tispark/latest/pull_integration_test.groovy b/jobs/pingcap/tispark/latest/pull_integration_test.groovy new file mode 100644 index 0000000000..4e304514ab --- /dev/null +++ b/jobs/pingcap/tispark/latest/pull_integration_test.groovy @@ -0,0 +1,39 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +// For trunk and latest release branches. +pipelineJob('pingcap/tispark/pull_integration_test') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC") + } + properties { + // priority(0) // 0 fast than 1 + githubProjectUrl("https://github.com/pingcap/tispark/") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/pingcap/tispark/latest/pull_integration_test.groovy") + scm { + git { + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/pipelines/pingcap/tispark/latest/pod-pull_integration_test.yaml b/pipelines/pingcap/tispark/latest/pod-pull_integration_test.yaml new file mode 100644 index 0000000000..a3a7eb4525 --- /dev/null +++ b/pipelines/pingcap/tispark/latest/pod-pull_integration_test.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: golang + image: "hub.pingcap.net/jenkins/centos7_golang-1.12:cached" + tty: true + resources: + requests: + memory: 12Gi + cpu: "4" + limits: + memory: 16Gi + cpu: "6" + env: + - name: DOCKER_HOST + value: tcp://localhost:2375 + - name: java + image: hub.pingcap.net/jenkins/centos7_golang-1.13_java:cached + tty: true + resources: + limits: + memory: 24Gi + cpu: "8" + env: + - name: DOCKER_HOST + value: tcp://localhost:2375 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/pingcap/tispark/latest/pull_integration_test.groovy b/pipelines/pingcap/tispark/latest/pull_integration_test.groovy new file mode 100644 index 0000000000..fc0ce4d8d7 --- /dev/null +++ b/pipelines/pingcap/tispark/latest/pull_integration_test.groovy @@ -0,0 +1,93 @@ +@Library('tipipeline') _ + + +# hello +final K8S_NAMESPACE = "jenkins-tispark" +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final POD_TEMPLATE_FILE = 'pipelines/pingcap/tispark/latest/pod-pull_integration_test.yaml' +final PARALLEL_NUMBER = 18 +final REFS = readJSON(text: params.JOB_SPEC).refs + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'java' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 90, unit: 'MINUTES') + } + stages { + stage('Checkout') { + options { timeout(time: 5, unit: 'MINUTES') } + steps { + dir(REFS.repo) { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + } + } + } + } + stage('Prepare') { + environment { + PARALLEL_NUMBER = '18' + } + steps { + dir(REFS.repo) { + sh label: 'split tests', script: ''' + find core/src -name '*Suite*' | grep -v 'MultiColumnPKDataTypeSuite|PartitionTableSuite' > test + shuf test -o test2 + mv test2 test + + # set when TEST_REGION_SIZE is not equal 'normal' + sed -i 's/# region-max-size = "2MB"/region-max-size = "2MB"/' config/tikv.toml + sed -i 's/# region-split-size = "1MB"/region-split-size = "1MB"/' config/tikv.toml" + cat config/tikv.toml + + # set when TEST_MODE is not equal 'simple' + find core/src -name '*MultiColumnPKDataTypeSuite*' >> test + + # set when TEST_TIFLASH is true + wget https://raw.githubusercontent.com/PingCAP-QE/ci/main/jenkins/pipelines/ci/tispark/tidb_config-for-tiflash-test.properties + + # set when TEST_SPARK_CATALOG is true + touch core/src/test/resources/tidb_config.properties + echo "spark.sql.catalog.tidb_catalog=org.apache.spark.sql.catalyst.catalog.TiCatalog" >> core/src/test/resources/tidb_config.properties + + sed -i 's|core/src/test/scala/||g' test + sed -i 's|/|.|g' test + sed -i 's|.scala||g' test + split test -n r/$PARALLEL_NUMBER test_unit_ -a 2 --numeric-suffixes=1 + + wget -O core/src/test/resources/log4j.properties https://raw.githubusercontent.com/PingCAP-QE/ci/main/jenkins/pipelines/ci/tispark/log4j-ci.properties + bash core/scripts/version.sh + bash core/scripts/fetch-test-data.sh + mv core/src/test core-test/src/ + bash tikv-client/scripts/proto.sh + ''' + } + dir(REFS.repo) { + script { + component.fetchAndExtractArtifact(FILE_SERVER_URL, 'tikv', REFS.base_ref, REFS.pulls[0].title, 'centos7/tikv-server.tar.gz', 'bin', trunkBranch="master", artifactVerify=true) + component.fetchAndExtractArtifact(FILE_SERVER_URL, 'pd', REFS.base_ref, REFS.pulls[0].title, 'centos7/pd-server.tar.gz', 'bin', trunkBranch="master", artifactVerify=true) + component.fetchAndExtractArtifact(FILE_SERVER_URL, 'tiflash', REFS.base_ref, REFS.pulls[0].title, 'centos7/tiflash.tar.gz', 'bin', trunkBranch="master", artifactVerify=true) + component.fetchAndExtractArtifact(FILE_SERVER_URL, 'tidb', REFS.base_ref, REFS.pulls[0].title, 'centos7/tidb-server.tar.gz', 'bin', trunkBranch="master", artifactVerify=true) + } + } + + } + } + stage('Tests') { + + } + } +} diff --git a/prow-jobs/kustomization.yaml b/prow-jobs/kustomization.yaml index 6de0a0571f..fa99345357 100644 --- a/prow-jobs/kustomization.yaml +++ b/prow-jobs/kustomization.yaml @@ -98,6 +98,7 @@ configMapGenerator: - pingcap_tiflow_release-8.5-presubmits.yaml=pingcap/tiflow/release-8.5-presubmits.yaml - pingcap_tiproxy_latest-postsubmits.yaml=pingcap/tiproxy/latest-postsubmits.yaml - pingcap_tiproxy_latest-presubmits.yaml=pingcap/tiproxy/latest-presubmits.yaml + - pingcap_tispark_latest-presubmits.yaml=pingcap/tispark/latest-presubmits.yaml - ti-community-infra_configs_periodics.yaml=ti-community-infra/configs/periodics.yaml - ti-community-infra_configs_postsubmits.yaml=ti-community-infra/configs/postsubmits.yaml - ti-community-infra_configs_presubmits.yaml=ti-community-infra/configs/presubmits.yaml diff --git a/prow-jobs/pingcap/tispark/latest-presubmits.yaml b/prow-jobs/pingcap/tispark/latest-presubmits.yaml new file mode 100644 index 0000000000..ba4f8d412d --- /dev/null +++ b/prow-jobs/pingcap/tispark/latest-presubmits.yaml @@ -0,0 +1,15 @@ +# struct ref: https://pkg.go.dev/sigs.k8s.io/prow/pkg/config#Presubmit +presubmits: + pingcap/tispark: + - name: pingcap/tispark/pull_integration_test + agent: jenkins + decorate: false # need add this. + skip_if_only_changed: "(\\.md|Dockerfile|OWNERS|OWNERS_ALIASES)$" + run_before_merge: true + optional: true # delete it when the job is stable + context: wip-pull-integration-test + skip_report: false + trigger: "(?m)^/test (?:.*? )?(integration-test)(?: .*?)?$" + rerun_command: "/test integration-test" + branches: + - .+