Skip to content

Commit

Permalink
Merge pull request #416 from dagood/dev/dagood/merge-b1.16-from-1.16
Browse files Browse the repository at this point in the history
[boring-1.16] Merge from 1.16
  • Loading branch information
dagood authored Feb 9, 2022
2 parents e47cf58 + 34b83cc commit faad8fc
Show file tree
Hide file tree
Showing 10 changed files with 160 additions and 34 deletions.
65 changes: 55 additions & 10 deletions eng/_core/archive/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,49 @@ import (
"io"
"io/fs"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"
)

// CreateFromSource runs a Git command to generate an archive file from the Go source code at
// "source". If output is "", the archive is produced in the build directory inside the
// "eng/artifacts/bin" directory. A checksum file is also produced.
func CreateFromSource(source string, output string) error {
fmt.Printf("---- Creating Go source archive (tarball) from '%v'...\n", source)

if output == "" {
output = filepath.Join(getBinDir(source), fmt.Sprintf("go.%v.src.tar.gz", getBuildID()))
}

// Ensure the target directory exists.
archiveDir := filepath.Dir(output)
if err := os.MkdirAll(archiveDir, os.ModeDir|os.ModePerm); err != nil {
return err
}

// Use "^{tree}" to avoid Git including a global extended pax header. The commit it would list
// is a temporary commit, and would only be confusing. See https://git-scm.com/docs/git-archive.
cmd := exec.Command("git", "archive", "-o", output, "--prefix=go/", "HEAD^{tree}")
cmd.Dir = source
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Printf("---- Running command: %v\n", cmd.Args)
if err := cmd.Run(); err != nil {
return err
}

fmt.Printf("---- Creating checksum file...\n")
if err := writeSHA256ChecksumFile(output); err != nil {
return err
}

fmt.Printf("---- Pack complete.\n")
return nil
}

// CreateFromBuild walks the Go build directory at "source" and produces an archive with the path
// "output". If output is "", CreateFromBuild produces a file in the build directory inside the
// "eng/artifacts/bin" directory. The output directory is created if it doesn't exist. This function
Expand All @@ -34,21 +71,14 @@ func CreateFromBuild(source string, output string) error {
fmt.Printf("---- Creating Go archive (zip/tarball) from '%v'...\n", source)

if output == "" {
// If BUILD_BUILDNUMBER is defined (e.g. a CI build), use it. For local builds, use "dev".
archiveVersion := os.Getenv("BUILD_BUILDNUMBER")
if archiveVersion == "" {
archiveVersion = "dev"
}

archiveVersion := getBuildID()
archiveExtension := ".tar.gz"
if runtime.GOOS == "windows" {
archiveExtension = ".zip"
}

archiveName := fmt.Sprintf("go.%v.%v-%v%v", archiveVersion, runtime.GOOS, runtime.GOARCH, archiveExtension)
binDir := filepath.Join(source, "..", "eng", "artifacts", "bin")

output = filepath.Join(binDir, archiveName)
output = filepath.Join(getBinDir(source), archiveName)
}

// Ensure the target directory exists.
Expand Down Expand Up @@ -186,12 +216,27 @@ func CreateFromBuild(source string, output string) error {
}

fmt.Printf("---- Creating checksum file...\n")
writeSHA256ChecksumFile(output)
if err := writeSHA256ChecksumFile(output); err != nil {
return err
}

fmt.Printf("---- Pack complete.\n")
return nil
}

// getBuildID returns BUILD_BUILDNUMBER if defined (e.g. a CI build). Otherwise, "dev".
func getBuildID() string {
archiveVersion := os.Getenv("BUILD_BUILDNUMBER")
if archiveVersion == "" {
return "dev"
}
return archiveVersion
}

func getBinDir(source string) string {
return filepath.Join(source, "..", "eng", "artifacts", "bin")
}

// getArchivePathRuntime takes a path like "go1.7.linux-amd64.tar.gz" and extension like ".tar.gz",
// and returns the os (linux) and arch (amd64). The "path" extension may have multiple '.'
// characters in it, so "ext" must be passed in explicitly or else the match would be ambiguous.
Expand Down
46 changes: 46 additions & 0 deletions eng/_core/cmd/pack-source/pack-source.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Copyright (c) Microsoft Corporation.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
"flag"
"fmt"
"os"
"path/filepath"

"github.com/microsoft/go/_core/archive"
)

const description = `
This command creates a source archive of the Go submodule at HEAD.
`

func main() {
output := flag.String("o", "", "The path of the archive file to create, including extension. Default: a tar.gz file including build number in 'eng/artifacts/bin'.")
help := flag.Bool("h", false, "Print this help message.")

flag.Usage = func() {
fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n")
flag.PrintDefaults()
fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description)
}

flag.Parse()
if *help {
flag.Usage()
return
}

repoRootDir, err := os.Getwd()
if err != nil {
panic(err)
}

goRootDir := filepath.Join(repoRootDir, "go")

if err := archive.CreateFromSource(goRootDir, *output); err != nil {
panic(err)
}
}
5 changes: 5 additions & 0 deletions eng/_core/cmd/submodule-refresh/submodule-refresh.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ applies patches to the stage by default, or optionally as commits.
`

var commits = flag.Bool("commits", false, "Apply the patches as commits.")
var skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.")
var origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.")
var shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.")
var fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.")
Expand Down Expand Up @@ -57,6 +58,10 @@ func refresh(rootDir string) error {
return err
}

if *skipPatch {
return nil
}

mode := patch.ApplyModeIndex
if *commits {
mode = patch.ApplyModeCommits
Expand Down
2 changes: 1 addition & 1 deletion eng/_util/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ module github.com/microsoft/go/_util
go 1.16

require (
github.com/microsoft/go-infra v0.0.0-20211206195537-520f2621bcf4
github.com/microsoft/go-infra v0.0.0-20220208232830-51df6c5b8843
gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d
)
4 changes: 2 additions & 2 deletions eng/_util/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/microsoft/go-infra v0.0.0-20211206195537-520f2621bcf4 h1:goYnnCGbN9HBwd0CtdiokVAlq5rTQkqq66K43kvGnhc=
github.com/microsoft/go-infra v0.0.0-20211206195537-520f2621bcf4/go.mod h1:3IVGTm7qFJldQHximiWLg2kYfmugjZMGNHnvUo5Mo5M=
github.com/microsoft/go-infra v0.0.0-20220208232830-51df6c5b8843 h1:IAk1GrsBP2l3sdWnaARrLALS9m6fVkGNgefpWRS0x2c=
github.com/microsoft/go-infra v0.0.0-20220208232830-51df6c5b8843/go.mod h1:3IVGTm7qFJldQHximiWLg2kYfmugjZMGNHnvUo5Mo5M=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down
3 changes: 3 additions & 0 deletions eng/pipeline/jobs/builders-to-jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,15 @@ parameters:
# If true, include a signing job that depends on all 'buildandpack' builder jobs finishing. This
# lets us start the lengthy tasks of signing and testing in parallel.
sign: false
# If true, generate source archive tarballs, and sign them if signing is enabled.
createSourceArchive: false

jobs:
- ${{ each builder in parameters.builders }}:
- template: run-job.yml
parameters:
builder: ${{ builder }}
createSourceArchive: ${{ parameters.createSourceArchive }}

- ${{ if eq(parameters.sign, true) }}:
- template: sign-job.yml
Expand Down
2 changes: 2 additions & 0 deletions eng/pipeline/jobs/go-builder-matrix-jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@ parameters:
innerloop: false
outerloop: false
sign: false
createSourceArchive: false

jobs:
- template: shorthand-builders-to-builders.yml
parameters:
jobsTemplate: builders-to-jobs.yml
jobsParameters:
sign: ${{ parameters.sign }}
createSourceArchive: ${{ parameters.createSourceArchive }}
shorthandBuilders:
- ${{ if eq(parameters.innerloop, true) }}:
- { os: linux, arch: amd64, config: buildandpack }
Expand Down
37 changes: 17 additions & 20 deletions eng/pipeline/jobs/run-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
parameters:
# { id, os, arch, config, distro? }
builder: {}
createSourceArchive: false

jobs:
- job: ${{ parameters.builder.id }}
Expand Down Expand Up @@ -41,34 +42,30 @@ jobs:
Write-Host "##vso[task.setvariable variable=GO_MAKE_MAX_RETRY_ATTEMPTS]5"
displayName: Increase 'make' retry attempts
# Initialize stage 0 toolset ahead of time so we can track timing data separately from the
# build operations. When we call this script again later, it won't download Go again.
- pwsh: |
. eng/utilities.ps1
Get-Stage0GoRoot
displayName: Init stage 0 Go toolset
- template: ../steps/init-submodule-task.yml

# Create the source archive on one job only. The os choice is arbitrary.
- ${{ if and(eq(parameters.createSourceArchive, true), eq(parameters.builder.config, 'buildandpack'), eq(parameters.builder.os, 'linux')) }}:
- pwsh: |
git config --global user.name "microsoft-golang-bot"
git config --global user.email "[email protected]"
# Turn the patches into commits, so HEAD includes the changes.
eng/run.ps1 submodule-refresh -commits
eng/run.ps1 pack-source
displayName: Archive submodule source
- pwsh: |
function fetch_submodule() {
eng/run.ps1 submodule-refresh -shallow @args
}
if ("$env:FETCH_BEARER_TOKEN") {
fetch_submodule `
-origin 'https://[email protected]/dnceng/internal/_git/microsoft-go-mirror' `
-fetch-bearer-token $env:FETCH_BEARER_TOKEN
} else {
fetch_submodule
}
# If non-public, use access token to fetch from repo. If public, don't use the access token,
# because anonymous auth is fine.
${{ if ne(variables['System.TeamProject'], 'public') }}:
env:
FETCH_BEARER_TOKEN: $(System.AccessToken)
displayName: Set up submodule from internal mirror
${{ if eq(variables['System.TeamProject'], 'public') }}:
displayName: Set up submodule
# Apply the patches as staged changes, so the HEAD commit is the same as upstream.
eng/run.ps1 submodule-refresh
displayName: Apply patches
# Use build script directly for "buildandpack". If we used run-builder, we would need to
# download its external module dependencies.
Expand Down
4 changes: 3 additions & 1 deletion eng/pipeline/rolling-internal-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ stages:
parameters:
innerloop: true
sign: true
createSourceArchive: true

- stage: Publish
dependsOn: Build
Expand All @@ -37,6 +38,7 @@ stages:
steps:
- template: steps/checkout-unix-task.yml
- template: steps/init-pwsh-task.yml
- template: steps/init-submodule-task.yml

- pwsh: |
function TrimStart($s, $prefix) {
Expand Down Expand Up @@ -68,7 +70,7 @@ stages:
- pwsh: |
eng/run.ps1 createbuildassetjson `
-artifacts-dir '$(Pipeline.Workspace)/Binaries Signed/' `
-source-dir '$(Build.SourcesDirectory)' `
-source-dir '$(Build.SourcesDirectory)/go' `
-destination-url '$(blobDestinationUrl)' `
-branch '$(PublishBranchAlias)' `
-o '$(Pipeline.Workspace)/Binaries Signed/assets.json'
Expand Down
26 changes: 26 additions & 0 deletions eng/pipeline/steps/init-submodule-task.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Copyright (c) Microsoft Corporation.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# Shallow checkout sources on Windows
steps:
- pwsh: |
function fetch_submodule() {
eng/run.ps1 submodule-refresh -shallow -skip-patch @args
}
if ("$env:FETCH_BEARER_TOKEN") {
fetch_submodule `
-origin 'https://[email protected]/dnceng/internal/_git/microsoft-go-mirror' `
-fetch-bearer-token $env:FETCH_BEARER_TOKEN
} else {
fetch_submodule
}
# If non-public, use access token to fetch from repo. If public, don't use the access token,
# because anonymous auth is fine.
${{ if ne(variables['System.TeamProject'], 'public') }}:
env:
FETCH_BEARER_TOKEN: $(System.AccessToken)
displayName: Set up submodule from internal mirror
${{ if eq(variables['System.TeamProject'], 'public') }}:
displayName: Set up submodule

0 comments on commit faad8fc

Please sign in to comment.