Skip to content

Commit

Permalink
update to profparse
Browse files Browse the repository at this point in the history
  • Loading branch information
pmurley committed Sep 18, 2022
1 parent 4789b42 commit 3ac2ee7
Show file tree
Hide file tree
Showing 7 changed files with 569 additions and 275 deletions.
4 changes: 2 additions & 2 deletions cmd/buildFileCoverage/buildFileCoverage.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,14 @@ func main() {
log.SetReportCaller(true)
log.Infof("Begin creating metadata structures by reading %s...", covFile)

MetaMap, err = pp.ReadCovMetadata(covFile)
MetaMap, _, err = pp.ReadCovMetadata(covFile)
if err != nil {
log.Fatal(err)
}

FileCovCounts = make(map[string]int)

sampleCovMap, err := pp.ReadFileToCovMap(covFile)
sampleCovMap, _, err := pp.ReadFileToCovMap(covFile)
if err != nil {
log.Fatal(err)
}
Expand Down
157 changes: 90 additions & 67 deletions cmd/correlate/correlate.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,67 +73,68 @@ var SiteCats map[string]CloudflareCategoryEntry

var ExcludeVector []bool

//var DirectoriesOfInterest = []string{
// "third_party/blink",
// "android_webview",
// "apps",
// "ash",
// "base",
// "build",
// "cc",
// "chrome",
// "chromecast",
// "chromeos",
// "cloud_print",
// "codelabs",
// "components",
// "content",
// "courgette",
// "crypto",
// "dbus",
// "device",
// "docs",
// "extensions",
// "fuchsia",
// "gin",
// "google_apis",
// "google_update",
// "gpu",
// "headless",
// "infra",
// "ios",
// "ipc",
// "jingle",
// "media",
// "mojo",
// "native_client",
// "native_client_sdk",
// "net",
// "out",
// "pdf",
// "ppapi",
// "printing",
// "remoting",
// "rlz",
// "sandbox",
// "services",
// "skia",
// "sql",
// "storage",
// "styleguide",
// "testing",
// "third_party",
// "tools",
// "ui",
// "url",
// "v8",
// "weblayer",
//}

var DirectoriesOfInterest = []string{
"third_party/blink",
"android_webview",
"apps",
"ash",
"base",
"build",
"cc",
"chrome",
"chromecast",
"chromeos",
"cloud_print",
"codelabs",
"components",
"content",
"courgette",
"crypto",
"dbus",
"device",
"docs",
"extensions",
"fuchsia",
"gin",
"google_apis",
"google_update",
"gpu",
"headless",
"infra",
"ios",
"ipc",
"jingle",
"media",
"mojo",
"native_client",
"native_client_sdk",
"net",
"out",
"pdf",
"ppapi",
"printing",
"remoting",
"rlz",
"sandbox",
"services",
"skia",
"sql",
"storage",
"styleguide",
"testing",
"third_party",
"tools",
"ui",
"url",
"v8",
"weblayer",
"net/websockets",
}

//var DirectoriesOfInterest = []string{
// "net/websockets",
//}

func main() {
var covFile string
var resultsPath string
Expand All @@ -147,7 +148,7 @@ func main() {
flag.StringVar(&outfile, "out", "file_coverage.csv",
"Path to output file csv")
flag.StringVar(&excludeBVFile, "excludeBV", "",
"BV file to exlcude set regions from analysis")
"BV file to exclude set regions from analysis")

flag.Parse()

Expand All @@ -164,18 +165,23 @@ func main() {
log.SetReportCaller(true)
log.Infof("Begin creating metadata structures by reading %s...", covFile)

MetaMap, err = pp.ReadCovMetadata(covFile)
var metaProps pp.CovMapProperties
MetaMap, metaProps, err = pp.ReadCovMetadata(covFile)
if err != nil {
log.Fatal(err)
}

log.Infof("MetaMap Files: %d, Funcs: %d, Regions: %d", metaProps.NumFiles, metaProps.NumFunctions, metaProps.NumRegions)

FileCovCounts = make(map[string]int)

sampleCovMap, err := pp.ReadFileToCovMap(covFile)
sampleCovMap, props, err := pp.ReadFileToCovMap(covFile)
if err != nil {
log.Fatal(err)
}

log.Infof("CovMap Files: %d, Funcs: %d, Regions: %d", props.NumFiles, props.NumFunctions, props.NumRegions)

files := 0
functions := 0
regions := 0
Expand Down Expand Up @@ -203,24 +209,41 @@ func main() {
}
}
BVIndexToCodeRegionMap = pp.GenerateBVIndexToCodeRegionMap(Structure, MetaMap)
log.Infof("BVIndexToCodeRegionMap Length: %d", len(BVIndexToCodeRegionMap))

// Build FilenameToBVIndices Map
FilenameToBVIndices = make(map[string]BVRange)
currentIndex := 0
start := 0
end := 0
for k, v := range Structure {
totalRegions := 0

fileNames := make([]string, 0)
for fileName := range Structure {
fileNames = append(fileNames, fileName)
}
sort.Strings(fileNames)

for _, fileName := range fileNames {
start = currentIndex
for _, numBlocks := range v {
currentIndex += numBlocks
funcNames := make([]string, 0)
for funcName := range Structure[fileName] {
funcNames = append(funcNames, funcName)
}

for _, funcName := range funcNames {
currentIndex += Structure[fileName][funcName]
}
end = currentIndex
FilenameToBVIndices[k] = BVRange{
FilenameToBVIndices[fileName] = BVRange{
Start: start,
End: end,
}
totalRegions += end - start
}

log.Infof("Total Files, Regions for FilenameToBVIndices: %d, %d", len(FilenameToBVIndices), totalRegions)

FileCoverage = make(map[string][]float64)

log.Infof("Finished parsing metadata")
Expand Down Expand Up @@ -344,9 +367,9 @@ func worker(taskChan chan Task, resultChan chan Result, wg *sync.WaitGroup) {
indices := FilenameToBVIndices[fileName]
fileCov := 0
for i := indices.Start; i < indices.End; i++ {
//if ExcludeVector[i] {
// continue
//}
if ExcludeVector[i] {
continue
}
for _, dir := range DirectoriesOfInterest {
if isDirMap[dir] {
totalBlocksPerDir[dir] += 1
Expand Down
162 changes: 162 additions & 0 deletions cmd/generateExclude/generateExclude.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
package main

import (
"encoding/csv"
"flag"
log "github.com/sirupsen/logrus"
pp "github.com/teamnsrg/profparse"
"io"
"math"
"os"
"strconv"
"sync"
)

type Task struct {
Path string
}

var CompleteCounter int

var Accumulator []bool
var AccumLock sync.Mutex

/*
File,Function,Region Number,Functions in File,Regions in File,Regions in Function,Times File Covered,Percent Times File Covered,Times Function Covered,Percent Times Function Covered,Times Region Covered,Percent Times Region Covered
../../apps/app_lifetime_monitor.cc,_ZN4apps18AppLifetimeMonitor11AddObserverEPNS0_8ObserverE,0,15,55,1,3999,1.0000,3999,1.0000,3999,1.0000
../../apps/app_lifetime_monitor.cc,_ZN4apps18AppLifetimeMonitor13NotifyAppStopERKNSt3__112basic_stringIcNS1_11char_traitsIcEENS1_9allocatorIcEEEE,1,15,55,2,3999,1.0000,0,0.0000,0,0.0000
../../apps/app_lifetime_monitor.cc,_ZN4apps18AppLifetimeMonitor13NotifyAppStopERKNSt3__112basic_stringIcNS1_11char_traitsIcEENS1_9allocatorIcEEEE,2,15,55,2,3999,1.0000,0,0.0000,0,0.0000
../../apps/app_lifetime_monitor.cc,_ZN4apps18AppLifetimeMonitor14NotifyAppStartERKNSt3__112basic_stringIcNS1_11char_traitsIcEENS1_9allocatorIcEEEE,3,15,55,2,3999,1.0000,0,0.0000,0,0.0000
../../apps/app_lifetime_monitor.cc,_ZN4apps18AppLifetimeMonitor14NotifyAppStartERKNSt3__112basic_stringIcNS1_11char_traitsIcEENS1_9allocatorIcEEEE,4,15,55,2,3999,1.0000,0,0.0000,0,0.0000
../../apps/app_lifetime_monitor.cc,_ZN4apps18AppLifetimeMonitor14RemoveObserverEPNS0_8ObserverE,5,15,55,1,3999,1.0000,3998,0.9997,3998,0.9997
../../apps/app_lifetime_monitor.cc,_ZN4apps18AppLifetimeMonitor16OnAppWindowShownEPN10extensions9AppWindowEb,6,15,55,8,3999,1.0000,0,0.0000,0,0.0000
../../apps/app_lifetime_monitor.cc,_ZN4apps18AppLifetimeMonitor16OnAppWindowShownEPN10extensions9AppWindowEb,7,15,55,8,3999,1.0000,0,0.0000,0,0.0000
../../apps/app_lifetime_monitor.cc,_ZN4apps18AppLifetimeMonitor16OnAppWindowShownEPN10extensions9AppWindowEb,8,15,55,8,3999,1.0000,0,0.0000,0,0.0000
*/

var COVERED_ALWAYS_OR_NEVER_EXCLUDE = []string{
"output/vanilla_region_coverage.csv",
"output/gremlins_region_coverage.csv",
// "output/headless_region_coverage.csv",
}

var COVERED_EVEN_ONCE_EXCLUDE = []string{
"aboutblank_region_coverage.csv",
}

const float64EqualityThreshold = 1e-6

func almostEqual(a, b float64) bool {
return math.Abs(a-b) <= float64EqualityThreshold
}

func main() {
var outfile string
var covFile string

flag.StringVar(&outfile, "out", "output/exclude_vector.bv",
"Path to output file csv")
flag.StringVar(&covFile, "coverage-file", "coverage.txt",
"Path to sample text coverage file for metadata generation")
flag.Parse()

var err error

sampleCovMap, _, err := pp.ReadFileToCovMap(covFile)
if err != nil {
log.Fatal(err)
}

files := 0
functions := 0
regions := 0

Structure := pp.ConvertCovMapToStructure(sampleCovMap)
for _, v1 := range Structure {
files += 1
for _, v2 := range v1 {
functions += 1
regions += v2
}
}

alwaysSoFar := make([]bool, regions)
neverSoFar := make([]bool, regions)
for i, _ := range alwaysSoFar {
alwaysSoFar[i] = true
neverSoFar[i] = true
}

for _, fileName := range COVERED_ALWAYS_OR_NEVER_EXCLUDE {
alwaysCovered, _ := pp.CountCoveredRegions(alwaysSoFar)
log.Infof("Always covered: %d", alwaysCovered)
neverCovered, _ := pp.CountCoveredRegions(neverSoFar)
log.Infof("Never covered: %d", neverCovered)

alwaysSoFar, neverSoFar, err = getAlwaysAndNeverCoveredVector(alwaysSoFar, neverSoFar, fileName, regions)
if err != nil {
log.Fatal(err)
}
}

excludeVector, totalExcluded, err := pp.CombineBVs([][]bool{
alwaysSoFar, neverSoFar,
})

log.Infof("Total Excluded: %d", totalExcluded)

pp.WriteFileFromBV(outfile, excludeVector)
alwaysCovered, _ := pp.CountCoveredRegions(alwaysSoFar)
log.Infof("Always covered: %d", alwaysCovered)
neverCovered, _ := pp.CountCoveredRegions(neverSoFar)
log.Infof("Never covered: %d", neverCovered)

totalExcludedRegions, totalRegions := pp.CountCoveredRegions(excludeVector)
log.Infof("Excluding a total of %d out of %d regions", totalExcludedRegions, totalRegions)
}

func getAlwaysAndNeverCoveredVector(alwaysSoFar []bool, neverSoFar []bool, regionCoverageFile string, numRegions int) ([]bool, []bool, error) {
f, err := os.Open(regionCoverageFile)
if err != nil {
return nil, nil, err
}
reader := csv.NewReader(f)

header := true

always := make([]bool, numRegions)
never := make([]bool, numRegions)

for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}

if header {
header = false
continue
}

regionNumber, err := strconv.Atoi(record[2])
if err != nil {
log.Fatal(err)
}

percentTimesRegionCovered, err := strconv.ParseFloat(record[11], 32)
if err != nil {
log.Fatal(err)
}

isNeverCovered := almostEqual(percentTimesRegionCovered, 0.0)
isAlwaysCovered := almostEqual(percentTimesRegionCovered, 1.0)

always[regionNumber] = alwaysSoFar[regionNumber] && isAlwaysCovered
never[regionNumber] = neverSoFar[regionNumber] && isNeverCovered
}

return always, never, nil
}
Loading

0 comments on commit 3ac2ee7

Please sign in to comment.