Skip to content

Commit

Permalink
faster and fewer errors
Browse files Browse the repository at this point in the history
  • Loading branch information
@michael1026
michael1026 committed May 24, 2022
1 parent 79c8844 commit ebd73c8
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 109 deletions.
134 changes: 40 additions & 94 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,15 @@ type Request struct {
}

type Response struct {
*http.Response
doc *goquery.Document
url string
}

type Body struct {
body string
url string
}

type FoundParameters struct {
parameters []string
url string
Expand Down Expand Up @@ -84,7 +89,7 @@ func main() {
sizeCheckReqChannel := make(chan Request)
readyToScanChannel := make(chan string)
parameterURLChannel := make(chan Request)
parameterRespChannel := make(chan Response)
parameterRespChannel := make(chan Body)
foundParametersChannel := make(chan FoundParameters)
wg := sync.WaitGroup{}

Expand Down Expand Up @@ -141,7 +146,7 @@ func writeJsonResults(foundParamsChan chan FoundParameters, outputFile string) {
}
}

func findReflections(parameterResponses chan Response, foundParamsChan chan FoundParameters) {
func findReflections(parameterResponses chan Body, foundParamsChan chan FoundParameters) {
var wg sync.WaitGroup

for i := 0; i < 10; i++ {
Expand All @@ -151,13 +156,7 @@ func findReflections(parameterResponses chan Response, foundParamsChan chan Foun

for resp := range parameterResponses {
if entry, ok := loadResults(resp.url); ok {
doc, err := goquery.NewDocumentFromResponse(resp.Response)

if err != nil {
continue
}

foundParams := reflectedscanner.CheckDocForReflections(doc, &entry)
foundParams := reflectedscanner.CheckDocForReflections(resp.body, &entry)

if len(foundParams) > 0 {
for _, param := range foundParams {
Expand All @@ -178,7 +177,7 @@ func findReflections(parameterResponses chan Response, foundParamsChan chan Foun
close(foundParamsChan)
}

func getParameterResponses(parameterURLs chan Request, parameterResponses chan Response) {
func getParameterResponses(parameterURLs chan Request, parameterResponses chan Body) {
var wg sync.WaitGroup

for i := 0; i < 10; i++ {
Expand All @@ -195,11 +194,14 @@ func getParameterResponses(parameterURLs chan Request, parameterResponses chan R

defer resp.Body.Close()

parameterResponses <- Response{
Response: resp,
url: req.url,
}
bodyString := util.ResponseToBodyString(resp)

if resp.StatusCode == http.StatusOK {
parameterResponses <- Body{
body: bodyString,
url: req.url,
}
}
}
}()
}
Expand Down Expand Up @@ -259,18 +261,19 @@ func checkURLStability(stabilityRespChannel chan Response, stableChannel chan st
defer close(stableChannel)

for resp := range stabilityRespChannel {
doc, err := goquery.NewDocumentFromResponse(resp.Response)

if entry, ok := loadResults(resp.url); ok {
if err != nil || doc == nil {
body, err := resp.doc.Html()

if err != nil {
fmt.Printf("%s is unstable. Skipping.\n", resp.url)
entry.Stable = false
addToResults(resp.url, entry)
continue
}

if entry.NumberOfCheckedURLs == 0 {
entry.PotentialParameters = findPotentialParameters(doc)
entry.CanaryCount = reflectedscanner.CountReflections(doc, entry.CanaryValue)
entry.PotentialParameters = findPotentialParameters(resp.doc)
entry.CanaryCount = reflectedscanner.CountReflections(body, entry.CanaryValue)
}

entry.NumberOfCheckedURLs++
Expand All @@ -279,14 +282,7 @@ func checkURLStability(stabilityRespChannel chan Response, stableChannel chan st
continue
}

if err != nil {
fmt.Printf("%s is unstable. Skipping.\n", resp.url)
entry.Stable = false
addToResults(resp.url, entry)
continue
}

if entry.CanaryCount != reflectedscanner.CountReflections(doc, entry.CanaryValue) {
if entry.CanaryCount != reflectedscanner.CountReflections(body, entry.CanaryValue) {
fmt.Printf("%s is unstable. Skipping.\n", resp.url)
entry.Stable = false
addToResults(resp.url, entry)
Expand Down Expand Up @@ -467,9 +463,15 @@ func getStabilityResponses(requests chan Request, responses chan Response) {
continue
}

responses <- Response{
url: req.url,
Response: resp,
defer resp.Body.Close()

doc, err := goquery.NewDocumentFromResponse(resp)

if err == nil && doc != nil {
responses <- Response{
url: req.url,
doc: doc,
}
}
}
}
Expand All @@ -491,21 +493,16 @@ func findPotentialParameters(doc *goquery.Document) map[string]string {
doc.Find("input").Each(func(index int, item *goquery.Selection) {
name, ok := item.Attr("name")

if ok && len(name) > 0 && len(name) < 20 {
if ok && len(name) > 0 && len(name) <= 15 {
parameters[name] = util.RandSeq(10)
}
})

regexWordlist := keywordsFromRegex(doc)

for _, word := range regexWordlist {
parameters[word] = util.RandSeq(10)
}

for _, word := range wordlist {
parameters[word] = util.RandSeq(10)
}

return parameters
}

Expand All @@ -523,7 +520,11 @@ func keywordsFromRegex(doc *goquery.Document) []string {
fmt.Printf("Error reading doc: %s\n", err)
}

regexs := [...]string{"\"[a-zA-Z_\\-]+\":", "[a-zA-Z_\\-]+:(\\d|{|\"|\\s)"}
regexs := [...]string{
"\"[a-zA-Z_\\-]{1,20}\":",
"'[a-zA-Z_\\-]{1,20}':",
"[a-zA-Z_\\-]{1,20}:({|\"|\\s)",
"[a-zA-Z_\\-]{1,20} = (\"|')"}

for _, regex := range regexs {
re := regexp.MustCompile(regex)
Expand All @@ -537,7 +538,7 @@ func keywordsFromRegex(doc *goquery.Document) []string {
match = strings.ReplaceAll(match, " ", "")

if match != "" {
newWordlist = util.AppendIfMissing(wordlist, match)
newWordlist = util.AppendIfMissing(newWordlist, match)
}
}
}
Expand All @@ -546,61 +547,6 @@ func keywordsFromRegex(doc *goquery.Document) []string {
return newWordlist
}

/***********************************************************************
*
* Calculates the max number of parameters before the page breaks
*
************************************************************************/

func calculateMaxParameters(scanInfo *scan.URLInfo, client *http.Client, rawUrl string) {
maxParameters := 100
parsedUrl, err := url.Parse(rawUrl)

if err != nil {
fmt.Printf("Error parsing URL: %s\n", err)
return
}

resp, err := client.Head(rawUrl)
if err != nil {
fmt.Printf("Error executing request: %s\n", err)
return
}

resp.Body.Close()

query := parsedUrl.Query()

for i := 0; i < 100; i++ {
query.Set(util.RandSeq(7), util.RandSeq(7))
}

for i := 0; i < 15; i++ {
for i := 0; i < 100; i++ {
query.Set(util.RandSeq(10), util.RandSeq(10))
}

parsedUrl.RawQuery = query.Encode()

resp, err = client.Head(parsedUrl.String())

if err != nil {
return
}

defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
scanInfo.MaxParams = maxParameters
return
}

maxParameters += 50
}

scanInfo.MaxParams = 1500
}

func addToResults(key string, info scan.URLInfo) {
resultsMutex.Lock()
results[key] = info
Expand Down
22 changes: 7 additions & 15 deletions reflectedscanner/reflectedscanner.go
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,24 @@
package reflectedscanner

import (
"fmt"
"strings"

"github.com/PuerkitoBio/goquery"
"github.com/michael1026/paramfinderSlimmed/types/scan"
"github.com/michael1026/paramfinderSlimmed/util"
)

func CheckStability(canary *string, doc *goquery.Document, urlInfo *scan.URLInfo) {
canaryCount := CountReflections(doc, *canary)
func CheckStability(canary *string, body string, urlInfo *scan.URLInfo) {
canaryCount := CountReflections(body, *canary)

if urlInfo.CanaryCount != canaryCount {
urlInfo.Stable = false
}
}

func CheckDocForReflections(doc *goquery.Document, urlInfo *scan.URLInfo) []string {
func CheckDocForReflections(body string, urlInfo *scan.URLInfo) []string {
var foundParameters []string

if CountReflections(doc, urlInfo.CanaryValue) != urlInfo.CanaryCount {
if CountReflections(body, urlInfo.CanaryValue) != urlInfo.CanaryCount {
// something happened with the response to cause the canary count to not be correct
// this is probably caused by a parameter included in the request
// for now, we are going to ignore this URL, but in the future, I'd like to find the parameter that caused this
Expand All @@ -29,7 +27,7 @@ func CheckDocForReflections(doc *goquery.Document, urlInfo *scan.URLInfo) []stri
}

for param, value := range urlInfo.PotentialParameters {
counted := CountReflections(doc, value)
counted := CountReflections(body, value)

if counted > urlInfo.CanaryCount {
foundParameters = util.AppendIfMissing(foundParameters, param)
Expand All @@ -50,12 +48,6 @@ func CheckDocForReflections(doc *goquery.Document, urlInfo *scan.URLInfo) []stri
return foundParameters
}

func CountReflections(doc *goquery.Document, canary string) int {
html, err := doc.Html()

if err != nil {
fmt.Printf("Error converting to HTML: %s\n", err)
}

return strings.Count(html, canary)
func CountReflections(body string, canary string) int {
return strings.Count(body, canary)
}
18 changes: 18 additions & 0 deletions util/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ package util
import (
"bytes"
"encoding/json"
"fmt"
"io"
"math/rand"
"net/http"
)

var letters = []rune("abcdefghijklmnopqrstuvwxyz")
Expand Down Expand Up @@ -55,3 +58,18 @@ func DeleteByKey(m *map[string]string, val string) {
}
}
}

func ResponseToBodyString(resp *http.Response) (body string) {
bodyString := ""

if resp.StatusCode == http.StatusOK {
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Printf("Error getting string %s\n", err)
return bodyString
}
bodyString = string(bodyBytes)
}

return bodyString
}

0 comments on commit ebd73c8

Please sign in to comment.