added subdomain center

README.md

@@ -52,6 +52,7 @@
   - **[Driftnet](https://driftnet.io)**
   - **[DayDayMap](https://www.daydaymap.com)**
   - **[NerdyData](https://www.nerdydata.com/?utm_source=projectdiscovery/uncover)**
+  - **[Subdomain Center](https://www.subdomain.center)**
 - Multiple API key input support
 - Automatic API key randomization
 - **stdin** / **stdout** support for input
@@ -179,6 +180,9 @@ daydaymap:
 nerdydata:
   - NERDYDATA_API_KEY_1
   - NERDYDATA_API_KEY_2
+subdomaincenter:
+  - SUBDOMAINCENTER_API_KEY_1
+  - SUBDOMAINCENTER_API_KEY_2
 ```
 
 When multiple keys/credentials are specified for same provider in the config file, random key will be used for each execution.
@@ -419,6 +423,31 @@ echo 8.8.8.8/20 | uncover -e driftnet
 ...
 ```
 
+### Subdomain Center keyword search
+
+**uncover** supports the [Subdomain Center](https://www.subdomain.center) `ammonites` engine, which finds hosts anywhere in the dataset carrying a given subdomain label — handy for turning up forgotten admin panels, VPN gateways or staging environments across the internet.
+
+The query is the label to search for. Scope it to a single zone with `domain:`, and widen it from an exact label match to a prefix match with `match:prefix`.
+
+```console
+uncover -subdomaincenter 'jenkins' -l 5 -silent
+
+prod.jenkins.dental2024-de.space
+www.jenkins.sayitright.space
+jenkins.conergy.us
+www.jenkins.dosugbarsmo.info
+qa.jenkins.black-friday-aanbiedingen.click
+```
+
+```console
+uncover -e subdomaincenter -q 'admin domain:hackerone.com match:prefix' -silent
+
+admin.hackerone.com
+admine.hackerone.com
+```
+
+An API key is optional here: without one the API answers from its anonymous tier with a capped sample of the result set, and with one it returns the full set, paginated.
+
 ### Field Format
 
 `-f, -field` flag can be used to indicate which fields to return, currently, `ip`, `port`, and `host` are supported and can be used to return desired fields.

integration-tests/integration-test.go

@@ -24,16 +24,17 @@ var (
 		"zoomeye": zoomeyeTestcases{},
 		"fofa":    fofaTestcases{},
 		//"hunter":  hunterTestcases{},
-		"quake":      quakeTestcases{},
-		"netlas":     netlasTestcases{},
-		"criminalip": criminalipTestcases{},
-		"hunterhow":  hunterhowTestcases{},
-		"google":     googleTestcases{},
-		"odin":       odinTestcases{},
-		"binaryedge": binaryedgeTestcases{},
-		"onyphe":     onypheTestcases{},
-		"greynoise":  greynoiseTestcases{},
-		"nerdydata":  nerdydataTestcases{},
+		"quake":           quakeTestcases{},
+		"netlas":          netlasTestcases{},
+		"criminalip":      criminalipTestcases{},
+		"hunterhow":       hunterhowTestcases{},
+		"google":          googleTestcases{},
+		"odin":            odinTestcases{},
+		"binaryedge":      binaryedgeTestcases{},
+		"onyphe":          onypheTestcases{},
+		"greynoise":       greynoiseTestcases{},
+		"nerdydata":       nerdydataTestcases{},
+		"subdomaincenter": subdomaincenterTestcases{},
 		// feature tests
 		"output": outputTestcases{},
 	}

integration-tests/source-test.go

@@ -333,3 +333,23 @@ func (h greynoiseTestcases) Execute() error {
 
 	return nil
 }
+
+type subdomaincenterTestcases struct{}
+
+func (h subdomaincenterTestcases) Execute() error {
+	// The API answers anonymously with a capped sample, so a key is optional
+	// here and only widens the result set.
+	if token := os.Getenv("SUBDOMAINCENTER_API_KEY"); token != "" {
+		subdomaincenterToken := fmt.Sprintf(`subdomaincenter: [%s]`, token)
+		_ = os.WriteFile(ConfigFile, []byte(subdomaincenterToken), 0644)
+		defer func() {
+			_ = os.RemoveAll(ConfigFile)
+		}()
+	}
+
+	results, err := testutils.RunUncoverAndGetResults(debug, "-subdomaincenter", "jenkins")
+	if err != nil {
+		return err
+	}
+	return expectResultsGreaterThanCount(results, 0)
+}

runner/options.go

@@ -65,6 +65,7 @@ type Options struct {
 	GreyNoise            goflags.StringSlice
 	Daydaymap            goflags.StringSlice
 	NerdyData            goflags.StringSlice
+	SubdomainCenter      goflags.StringSlice
 	DisableUpdateCheck   bool
 }
 
@@ -76,7 +77,7 @@ func ParseOptions() *Options {
 
 	flagSet.CreateGroup("input", "Input",
 		flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions),
-		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet,greynoise,daydaymap,nerdydata) (default shodan)", goflags.FileNormalizedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet,greynoise,daydaymap,nerdydata,subdomaincenter) (default shodan)", goflags.FileNormalizedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.AwesomeSearchQueries, "awesome-search-queries", "asq", nil, "use awesome search queries to discover exposed assets on the internet (example: -asq 'jira')", goflags.FileStringSliceOptions),
 	)
 
@@ -100,6 +101,7 @@ func ParseOptions() *Options {
 		flagSet.StringSliceVarP(&options.GreyNoise, "greynoise", "gn", nil, "search query for greynoise (example: -greynoise 'query.txt')", goflags.FileStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Daydaymap, "daydaymap", "ddm", nil, "search query for daydaymap (example: -daydaymap 'query.txt')", goflags.FileStringSliceOptions),
 		flagSet.StringSliceVarP(&options.NerdyData, "nerdydata", "nd", nil, "search query for NerdyData (example: -nerdydata 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.SubdomainCenter, "subdomaincenter", "sdc", nil, "subdomain label to search for with subdomain center (example: -subdomaincenter 'vpn domain:example.com')", goflags.FileStringSliceOptions),
 	)
 
 	flagSet.CreateGroup("config", "Config",
@@ -178,7 +180,8 @@ func ParseOptions() *Options {
 		len(options.Driftnet),
 		len(options.GreyNoise),
 		len(options.Daydaymap),
-		len(options.NerdyData)) {
+		len(options.NerdyData),
+		len(options.SubdomainCenter)) {
 		options.Engine = append(options.Engine, "shodan")
 	}
 
@@ -253,7 +256,8 @@ func (options *Options) validateOptions() error {
 		len(options.Driftnet),
 		len(options.GreyNoise),
 		len(options.Daydaymap),
-		len(options.NerdyData)) {
+		len(options.NerdyData),
+		len(options.SubdomainCenter)) {
 		return errors.New("no query provided")
 	}
 
@@ -283,7 +287,8 @@ func (options *Options) validateOptions() error {
 		len(options.Driftnet),
 		len(options.GreyNoise),
 		len(options.Daydaymap),
-		len(options.NerdyData)) {
+		len(options.NerdyData),
+		len(options.SubdomainCenter)) {
 		return errors.New("no engine specified")
 	}
 
@@ -330,6 +335,7 @@ func appendAllQueries(options *Options) {
 	appendQuery(options, "greynoise", options.GreyNoise...)
 	appendQuery(options, "daydaymap", options.Daydaymap...)
 	appendQuery(options, "nerdydata", options.NerdyData...)
+	appendQuery(options, "subdomaincenter", options.SubdomainCenter...)
 }
 
 func (options *Options) useAwesomeSearchQueries(awesomeSearchQueries []string) error {

sources/agent/subdomaincenter/subdomaincenter.go

@@ -0,0 +1,180 @@
+// Package subdomaincenter queries the Subdomain Center ammonites engine, which
+// finds hosts anywhere in the dataset carrying a given subdomain label.
+package subdomaincenter
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/projectdiscovery/uncover/sources"
+)
+
+// pageSize is the largest page an authenticated query asks for. The API accepts
+// an explicit limit of up to 1,000,000, but smaller pages complete faster and
+// are cheaper to retry individually.
+const pageSize = 10000
+
+type Agent struct{}
+
+func (agent *Agent) Name() string {
+	return "subdomaincenter"
+}
+
+func (agent *Agent) Query(ctx context.Context, session *sources.Session, query *sources.Query) (chan sources.Result, error) {
+	ammonitesRequest, err := newRequest(query.Query)
+	if err != nil {
+		return nil, err
+	}
+
+	results := make(chan sources.Result)
+
+	go func() {
+		defer close(results)
+
+		apiKey := session.Keys.SubdomainCenter
+		numberOfResults := 0
+
+		for offset := 0; ; {
+			page, err := agent.queryPage(ctx, session, ammonitesRequest, apiKey, offset, query.Limit-numberOfResults)
+			if err != nil {
+				sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err})
+				return
+			}
+
+			for _, host := range page.hosts {
+				if numberOfResults >= query.Limit {
+					return
+				}
+				raw, _ := json.Marshal(host)
+				if !sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Host: host, Raw: raw}) {
+					return
+				}
+				numberOfResults++
+			}
+
+			// The anonymous tier ignores limit/offset and always answers with a
+			// single capped sample, so there is nothing to page through.
+			if apiKey == "" || !page.truncated || len(page.hosts) == 0 {
+				return
+			}
+			offset = page.nextOffset
+		}
+	}()
+
+	return results, nil
+}
+
+// page is one response from the ammonites engine.
+type page struct {
+	hosts      []string
+	truncated  bool
+	nextOffset int
+}
+
+func (agent *Agent) queryPage(ctx context.Context, session *sources.Session, ammonitesRequest *request, apiKey string, offset, remaining int) (*page, error) {
+	limit := pageSize
+	if remaining > 0 && remaining < limit {
+		limit = remaining
+	}
+
+	httpRequest, err := sources.NewHTTPRequest(ctx, http.MethodGet, ammonitesRequest.buildURL(apiKey != "", offset, limit), nil)
+	if err != nil {
+		return nil, err
+	}
+	httpRequest.Header.Set("Accept", "application/json")
+	if apiKey != "" {
+		// Header only: the API rejects the key as a query parameter so it cannot
+		// leak through proxy, browser or CDN logs.
+		httpRequest.Header.Set("X-API-Key", apiKey)
+	}
+
+	resp, err := session.Do(httpRequest, agent.Name())
+	if err != nil {
+		if resp != nil {
+			_ = resp.Body.Close()
+		}
+		return nil, err
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	var hosts []string
+	if err := json.NewDecoder(resp.Body).Decode(&hosts); err != nil {
+		return nil, err
+	}
+
+	current := &page{
+		hosts:      hosts,
+		truncated:  strings.EqualFold(resp.Header.Get("X-Truncated"), "true"),
+		nextOffset: offset + len(hosts),
+	}
+	if next, err := strconv.Atoi(resp.Header.Get("X-Next-Offset")); err == nil && next > offset {
+		current.nextOffset = next
+	}
+
+	return current, nil
+}
+
+// request is an ammonites keyword search.
+type request struct {
+	keyword string
+	domain  string
+	match   string
+}
+
+// newRequest parses an uncover query into an ammonites request. The query is the
+// subdomain label to search for, optionally scoped to a single zone and widened
+// to a prefix search: "vpn", "vpn domain:example.com", "vpn match:prefix".
+func newRequest(query string) (*request, error) {
+	parsed := &request{}
+	for _, field := range strings.Fields(query) {
+		name, value, found := strings.Cut(field, ":")
+		if !found {
+			if parsed.keyword == "" {
+				parsed.keyword = field
+			}
+			continue
+		}
+		switch strings.ToLower(name) {
+		case "keyword":
+			parsed.keyword = value
+		case "domain":
+			parsed.domain = value
+		case "match":
+			parsed.match = value
+		default:
+			return nil, fmt.Errorf("subdomaincenter: unknown query field %q", name)
+		}
+	}
+
+	if len(parsed.keyword) < 2 {
+		return nil, errors.New("subdomaincenter: keyword must be at least 2 characters")
+	}
+
+	return parsed, nil
+}
+
+func (r *request) buildURL(authenticated bool, offset, limit int) string {
+	query := url.Values{}
+	query.Set("engine", "ammonites")
+	query.Set("keyword", r.keyword)
+	if r.domain != "" {
+		query.Set("domain", r.domain)
+	}
+	if r.match != "" {
+		query.Set("match", r.match)
+	}
+	if authenticated {
+		query.Set("limit", strconv.Itoa(limit))
+		query.Set("offset", strconv.Itoa(offset))
+	}
+
+	return "https://api.subdomain.center/?" + query.Encode()
+}

sources/keys.go

@@ -22,6 +22,7 @@ type Keys struct {
 	GreyNoiseKey    string
 	Daydaymap       string
 	NerdyDataToken  string
+	SubdomainCenter string
 }
 
 func (keys Keys) Empty() bool {
@@ -45,5 +46,6 @@ func (keys Keys) Empty() bool {
 		keys.DriftnetToken == "" &&
 		keys.GreyNoiseKey == "" &&
 		keys.Daydaymap == "" &&
-		keys.NerdyDataToken == ""
+		keys.NerdyDataToken == "" &&
+		keys.SubdomainCenter == ""
 }

sources/provider.go

@@ -39,6 +39,9 @@ type Provider struct {
 	GreyNoise  []string `yaml:"greynoise"`
 	Daydaymap  []string `yaml:"daydaymap"`
 	NerdyData  []string `yaml:"nerdydata"`
+	// SubdomainCenter is optional: without a key the API answers from its
+	// anonymous tier with a capped sample of the result set.
+	SubdomainCenter []string `yaml:"subdomaincenter"`
 }
 
 // NewProvider loads provider keys from default location and env variables
@@ -131,6 +134,9 @@ func (provider *Provider) GetKeys() Keys {
 	if len(provider.NerdyData) > 0 {
 		keys.NerdyDataToken = provider.NerdyData[rand.Intn(len(provider.NerdyData))]
 	}
+	if len(provider.SubdomainCenter) > 0 {
+		keys.SubdomainCenter = provider.SubdomainCenter[rand.Intn(len(provider.SubdomainCenter))]
+	}
 
 	return keys
 }
@@ -183,6 +189,7 @@ func (provider *Provider) LoadProviderKeysFromEnv() {
 	provider.Onyphe = appendIfExists(provider.Onyphe, "ONYPHE_API_KEY")
 	provider.GreyNoise = appendIfExists(provider.GreyNoise, "GREYNOISE_API_KEY")
 	provider.NerdyData = appendIfExists(provider.NerdyData, "NERDYDATA_API_KEY")
+	provider.SubdomainCenter = appendIfExists(provider.SubdomainCenter, "SUBDOMAINCENTER_API_KEY")
 }
 
 // HasKeys returns true if at least one agent/source has keys
@@ -206,6 +213,7 @@ func (provider *Provider) HasKeys() bool {
 		len(provider.GreyNoise) > 0,
 		len(provider.Daydaymap) > 0,
 		len(provider.NerdyData) > 0,
+		len(provider.SubdomainCenter) > 0,
 	)
 }