first commit

ZhuriLab · Mar 28, 2022 · 82b295d · 82b295d
commit 82b295d
Show file tree

Hide file tree

Showing 91 changed files with 115,659 additions and 0 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,60 @@
+name: 🎉 Build Binary
+on:
+  create:
+    tags:
+      - v*
+  workflow_dispatch:
+jobs:
+
+  build:
+    name: Build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up libpcap-dev
+        if: matrix.os == 'ubuntu-latest'
+        run: sudo apt-get install libpcap-dev gcc -y
+
+      - name: Get dependencies
+        run: go mod download
+
+      - name: Build On Linux
+        run: |
+          go build -o Starmap-linux cmd/Starmap.go
+          chmod +x Starmap-linux
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          GOENABLE: 1
+          CGO_LDFLAGS: "-Wl,-static -L/usr/lib/x86_64-linux-gnu/libpcap.a -lpcap -Wl,-Bdynamic"
+      - name: Build On Darwin
+        run: |
+          go build -o Starmap-darwin cmd/Starmap.go
+          chmod +x Starmap-darwin
+        if: matrix.os == 'macos-latest'
+      - name: Build On Windows
+        run: |
+          go build -o Starmap-windows.exe cmd/Starmap.go
+        if: matrix.os == 'windows-latest'
+        env:
+          GOOS: windows
+          GOENABLE: 1
+      - name: Release
+        uses: softprops/action-gh-release@master
+        with:
+          files: Starmap-*
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,10 @@
+# Binaries for programs and plugins
+*.db
+.idea
+.DS_Store
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,10 @@
+# Changelog
+
+
+## v0.0.3
+- 参考 [subjack](https://github.com/haccer/subjack) 添加子域名接管检测
+- 合并 [subfinder](https://github.com/projectdiscovery/subfinder) v2.5.0
+
+## v0.0.2
+- [subfinder](https://github.com/projectdiscovery/subfinder) 和 [ksubdomain](https://github.com/boy-hack/ksubdomain) 初步融合
+
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 ZhuriLab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/Makefile b/Makefile
@@ -0,0 +1,21 @@
+# Go build flags
+LDFLAGS=-ldflags "-s -w"
+
+default:
+	go build ${LDFLAGS} -o "Starmap" main.go
+
+# Compile Server - Windows x64
+windows:
+	export GOOS=windows;export GOARCH=amd64;go build ${LDFLAGS} -o "Starmap.exe" cmd/Starmap.go
+
+# Compile Server - Linux x64
+linux:
+	export GOOS=linux;export GOARCH=amd64;go build ${LDFLAGS} -o "Starmap" cmd/Starmap.go
+
+# Compile Server - Darwin x64
+darwin:
+	export GOOS=darwin;export GOARCH=amd64;go build ${LDFLAGS} -o "Starmap" cmd/Starmap.go
+
+# clean
+clean:
+	rm -rf ${DIR}
diff --git a/README.md b/README.md
@@ -0,0 +1,183 @@
+# 🌟 Starmap
+ 以 subfinder 为基础，融合 ksubdomain、 Amass 的一些优点进行二次开发的一款子域名收集工具，并增加了子域名接管检测功能。可以很方便作为 go 库集成进入项目中。
+
+- [Amass](https://github.com/OWASP/Amass/) 虽然搜集的方法多，但太笨重，不方便集成，目标多了会内存爆炸
+- [subfinder](https://github.com/projectdiscovery/subfinder) 非常方便集成，但是只有被动的方式
+- [ksubdomain](https://github.com/boy-hack/ksubdomain) 仅主动爆破，以及验证
+
+# 🍺 Installation
+下载二进制 https://github.com/ZhuriLab/Starmap/releases
+
+安装`libpcap`环境 
+- Windows 下载 npcap 驱动: https://npcap.com/#download (ksubdomain 推荐下载的winpcap驱动存在一点问题，我在虚拟机中跑不出任何东西，改用 npcap 驱动可以)
+- Linux 已经静态编译打包`libpcap`，无需其他操作
+- MacOS 自带`libpcap`,无需其他操作 
+
+# 🔅 Usage
+```
+Flags:
+INPUT:
+     -d, -domain string[]  domains to find subdomains for
+     枚举的目标域名
+     -dL, -list string  file containing list of domains for subdomain discovery
+     枚举的域名列表的文件
+
+SOURCE:
+     -s, -sources string[]  specific sources to use for discovery (-s crtsh,github)
+     被动使用的源
+     -recursive  use only recursive sources
+     仅使用递归源
+     -all  Use all sources (slow) for enumeration
+     使用所有源进行枚举
+     -es, -exclude-sources string[]  sources to exclude from enumeration (-es archiveis,zoomeye)
+      被动枚举中排除使用的源列表
+
+OUTPUT:
+     -o, -output string  file to write output to
+     输出文件名
+     -oJ, -json  write output in JSONL(ines) format
+     Json格式输出，该选项输出内容丰富
+
+CONFIGURATION:
+     -config string  flag config file
+     自定义API密钥等的配置文件位置 (default "/Users/yhy/.config/Starmap/config.yaml")
+     -nW, -active  display active subdomains only
+     仅显示活动子域
+     -proxy string  http proxy to use with subfinder
+     指定被动api获取子域名时的代理
+
+DEBUG:
+     -silent  show only subdomains in output
+     使用后屏幕将仅输出结果域名
+     -version  show version of Starmap
+     输出当前版本
+     -v  show verbose output
+     显示详细输出
+
+DNS BRUTE FORCING SUBDOMAIN:
+     -w string  Path to a different wordlist file for brute forcing
+     dns 爆破使用的字典
+     -ld string  Multilevel subdomain dictionary(level > 2 use)
+     dns 枚举多级域名的字典文件，当level大于2时候使用，不填则会默认
+     -l int  Number of blasting subdomain layers
+     枚举几级域名，默认为二级域名 (default 2)
+     -n int  Number of DNS forced subdomains
+     dns爆破每个域名的次数，默认跑一次 (default 1)
+     -b  Use DNS brute forcing subdomain(default true)
+     被动加 dns 主动爆破(默认使用) (default true)
+     -verify  DNS authentication survival, Export only verified domain names
+     验证被动获取的域名，使用后仅输出验证存活的域名
+     -dns string  DNS server, cn:China dns, in:International, all:(cn+in DNS),Select according to the target.
+     DNS服务器，默认国内的服务器(cn)(cn: 表示使用国内的 dns, in:国外 dns，all: 全部内置 dns, 根据目标选择 (default "cn")
+
+SUBDOMAIN TAKEOVER:
+     -takeover   Scan subdomain takeover (default False).
+     子域名接管检测 (默认：false)
+     -sa  subdomain take over: Request to test each URL (by default, only the URL matching CNAME is requested to test).
+     子域名接管检测：请求测试每个URL（默认情况下，仅请求测试与CNAME匹配的URL）
+```
+
+
+# 🎉 Starmap Go library
+
+```go
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/ZhuriLab/Starmap/pkg/passive"
+	"github.com/ZhuriLab/Starmap/pkg/resolve"
+	"github.com/ZhuriLab/Starmap/pkg/runner"
+	"io"
+	"io/ioutil"
+	"log"
+)
+
+// 作为 go library 集成
+func main() {
+
+	config, _ := runner.UnmarshalRead("/Users/yhy/.config/Starmap/config.yaml")
+
+	config.Recursive = resolve.DefaultResolvers
+	config.Sources = passive.DefaultSources
+	config.AllSources = passive.DefaultAllSources
+	config.Recursive = passive.DefaultRecursiveSources
+
+	runnerInstance, err := runner.NewRunner(&runner.Options{
+		Threads:            10, // Thread controls the number of threads to use for active enumerations
+		Timeout:            30, // Timeout is the seconds to wait for sources to respond
+		MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
+		Resolvers:          resolve.DefaultResolvers, // Use the default list of resolvers by marshaling it to the config
+		Sources:            passive.DefaultSources, // Use the default list of passive sources
+		AllSources:         passive.DefaultAllSources, // Use the default list of all passive sources
+		Recursive:          passive.DefaultRecursiveSources,	// Use the default list of recursive sources
+
+		YAMLConfig:         config,	// 读取自定义配置文件
+		All: 				true,
+		Verbose: 			false,
+		Brute:				true,
+		Verify:             true,	// 验证找到的域名
+		Silent: 			false,	// 是否为静默模式，只输出找到的域名
+		DNS: 				"cn",	// dns 服务器区域选择，根据目标选择不同区域得到的结果不同，国内网站的话，选择 cn，dns 爆破结果比较多
+		BruteWordlist:      "",		// 爆破子域的域名字典，不填则使用内置的
+		Level: 				2,		// 枚举几级域名，默认为二级域名
+		LevelDic:           "",		// 枚举多级域名的字典文件，当level大于2时候使用，不填则会默认
+		Takeover: 			false,	// 子域名接管检测
+		SAll: 				false,  // 子域名接管检测中请求全部 url，默认只对匹配的 cname 进行检测
+
+	})
+
+
+	buf := bytes.Buffer{}
+	err, subdomains := runnerInstance.EnumerateSingleDomain(context.Background(), "baidu.com", []io.Writer{&buf})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+
+	data, err := ioutil.ReadAll(&buf)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// 只输出域名
+	fmt.Printf("%s", data)
+
+	// 输出详细信息
+	/*
+		Host   	string 		`json:"host"`
+		Source 	string 		`json:"source"`
+		Ips    	[]string	`json:"ips"`
+		CNames  []string	`json:"cnames"`
+		TakeOver 	bool		`json:"take_over"`
+	*/
+	for _, result := range subdomains {
+		fmt.Println(result.Source, result.Host, result.Ips, result.CNames, result.TakeOver)
+	}
+}
+
+
+```
+
+# 📌 TODO
+
+- [ ] [Amass](https://github.com/OWASP/Amass/) 中的子域名检测技术
+- [x] 子域名接管检测
+
+# 💡 Tips
+ - 指定不同的 dns ，获取到的结果会不同。比如：如果目标是国内的网站，选择国内的 dns 得到的子域名结果可能会比较多
+
+
+# 👀 参考
+- [subfinder](https://github.com/projectdiscovery/subfinder)
+- [ksubdomain](https://github.com/boy-hack/ksubdomain)
+- [Amass](https://github.com/OWASP/Amass)
+
+#  📄 免责声明
+本工具仅面向合法授权的企业安全建设行为，在使用本工具进行检测时，您应确保该行为符合当地的法律法规，并且已经取得了足够的授权。
+
+如您在使用本工具的过程中存在任何非法行为，您需自行承担相应后果，作者将不承担任何法律及连带责任。
+
+在使用本工具前，请您务必审慎阅读、充分理解各条款内容，限制、免责条款或者其他涉及您重大权益的条款可能会以加粗、加下划线等形式提示您重点注意。 除非您已充分阅读、完全理解并接受本协议所有条款，否则，请您不要使用本工具。您的使用行为或者您以其他任何明示或者默示方式表示接受本协议的，即视为您已阅读并同意本协议的约束。
diff --git a/cmd/Starmap.go b/cmd/Starmap.go
@@ -0,0 +1,22 @@
+package main
+
+import (
+	"context"
+	"github.com/ZhuriLab/Starmap/pkg/runner"
+	"github.com/projectdiscovery/gologger"
+)
+
+func main() {
+	// Parse the command line flags and read config files
+	options := runner.ParseOptions()
+
+	newRunner, err := runner.NewRunner(options)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
+	}
+
+	err = newRunner.RunEnumeration(context.Background())
+	if err != nil {
+		gologger.Fatal().Msgf("Could not run enumeration: %s\n", err)
+	}
+}
diff --git a/go.mod b/go.mod
@@ -0,0 +1,50 @@
+module github.com/ZhuriLab/Starmap
+
+go 1.17
+
+require (
+	github.com/corpix/uarand v0.1.1
+	github.com/google/gopacket v1.1.19
+	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
+	github.com/json-iterator/go v1.1.12
+	github.com/lib/pq v1.10.4
+	github.com/logrusorgru/aurora v2.0.3+incompatible
+	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
+	github.com/pkg/errors v0.9.1
+	github.com/projectdiscovery/chaos-client v0.2.0
+	github.com/projectdiscovery/dnsx v1.0.7
+	github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c
+	github.com/projectdiscovery/goflags v0.0.7
+	github.com/projectdiscovery/gologger v1.1.4
+	github.com/rs/xid v1.3.0
+	github.com/spyse-com/go-spyse v1.2.4
+	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
+	github.com/valyala/fasthttp v1.34.0
+	go.uber.org/ratelimit v0.2.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+)
+
+require (
+	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
+	github.com/andybalholm/brotli v1.0.4 // indirect
+	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
+	github.com/karrick/godirwalk v1.16.1 // indirect
+	github.com/klauspost/compress v1.15.0 // indirect
+	github.com/miekg/dns v1.1.43 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect
+	github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc // indirect
+	github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect
+	github.com/projectdiscovery/mapcidr v0.0.8 // indirect
+	github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d // indirect
+	github.com/projectdiscovery/retryablehttp-go v1.0.2 // indirect
+	github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)