added option for -dns and -tldplus1

Author: lanrat

certgraph.go

@@ -10,13 +10,13 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lanrat/certgraph/dns"
 	"github.com/lanrat/certgraph/driver"
 	"github.com/lanrat/certgraph/driver/crtsh"
 	"github.com/lanrat/certgraph/driver/google"
 	"github.com/lanrat/certgraph/driver/http"
 	"github.com/lanrat/certgraph/driver/smtp"
 	"github.com/lanrat/certgraph/graph"
-	"github.com/lanrat/certgraph/status"
 )
 
 var (
@@ -42,7 +42,8 @@ var config struct {
 	cdn                 bool
 	maxSANsSize         int
 	tldPlus1            bool
-	checkNS             bool
+	updatePSL           bool
+	checkDNS            bool
 	printVersion        bool
 }
 
@@ -56,8 +57,9 @@ func init() {
 	flag.BoolVar(&config.includeCTExpired, "ct-expired", false, "include expired certificates in certificate transparency search")
 	flag.IntVar(&config.maxSANsSize, "sanscap", 80, "maximum number of uniq TLD+1 domains in certificate to include, 0 has no limit")
 	flag.BoolVar(&config.cdn, "cdn", false, "include certificates from CDNs")
-	flag.BoolVar(&config.checkNS, "ns", false, "check for NS records to determine if domain is registered")
+	flag.BoolVar(&config.checkDNS, "dns", false, "check for DNS records to determine if domain is registered")
 	flag.BoolVar(&config.tldPlus1, "tldplus1", false, "for every domain found, add tldPlus1 of the domain's parent")
+	flag.BoolVar(&config.updatePSL, "updatepsl", false, "Update the default Public Suffix List")
 	flag.UintVar(&config.maxDepth, "depth", 5, "maximum BFS depth to go")
 	flag.UintVar(&config.parallel, "parallel", 10, "number of certificates to retrieve in parallel")
 	flag.BoolVar(&config.details, "details", false, "print details about the domains crawled")
@@ -91,14 +93,23 @@ func main() {
 		return
 	}
 
+	// update the public suffix list if required
+	if config.updatePSL {
+		err := dns.UpdatePublicSuffixList(config.timeout)
+		if err != nil {
+			e(err)
+			return
+		}
+	}
+
 	// add domains passed to startDomains
 	startDomains := make([]string, 0, 1)
 	for _, domain := range flag.Args() {
 		d := strings.ToLower(domain)
 		if len(d) > 0 {
 			startDomains = append(startDomains, cleanInput(d))
 			if config.tldPlus1 {
-				tldPlus1, err := status.TLDPlus1(domain)
+				tldPlus1, err := dns.TLDPlus1(domain)
 				if err != nil {
 					continue
 				}
@@ -162,7 +173,9 @@ func v(a ...interface{}) {
 }
 
 func e(a ...interface{}) {
-	fmt.Fprintln(os.Stderr, a...)
+	if a != nil {
+		fmt.Fprintln(os.Stderr, a...)
+	}
 }
 
 // prints the graph as a json object
@@ -232,7 +245,7 @@ func breathFirstSearch(roots []string) {
 						wg.Add(1)
 						domainNodeInputChan <- graph.NewDomainNode(neighbor, domainNode.Depth+1)
 						if config.tldPlus1 {
-							tldPlus1, err := status.TLDPlus1(neighbor)
+							tldPlus1, err := dns.TLDPlus1(neighbor)
 							if err != nil {
 								continue
 							}
@@ -254,23 +267,7 @@ func breathFirstSearch(roots []string) {
 			domainNode, more := <-domainNodeOutputChan
 			if more {
 				if !config.printJSON {
-					if config.details {
-						fmt.Fprintln(os.Stdout, domainNode)
-					} else {
-						fmt.Fprintln(os.Stdout, domainNode.Domain)
-					}
-					if config.checkNS {
-						// TODO these ns lookups are likely done a LOT for many subdomains of the same domain
-						ns, err := status.HasNameservers(domainNode.Domain, config.timeout)
-						if err != nil {
-							v("NS check error:", domainNode.Domain, err)
-							continue
-						}
-						if !ns {
-							// TODO print tldplus1 in a good way
-							fmt.Fprintf(os.Stdout, "Missing NS: %s\n", domainNode.Domain)
-						}
-					}
+					printNode(domainNode)
 				} else if config.details {
 					fmt.Fprintln(os.Stderr, domainNode)
 				}
@@ -286,8 +283,16 @@ func breathFirstSearch(roots []string) {
 	<-done // wait for save to finish
 }
 
-// visit visit each node and get and set its neighbors
+// visit visits each node and get and set its neighbors
 func visit(domainNode *graph.DomainNode) {
+	// check NS if necessary
+	if config.checkDNS {
+		_, err := domainNode.CheckForDNS(config.timeout)
+		if err != nil {
+			v("CheckForNS", err)
+		}
+	}
+
 	// perform cert search
 	// TODO do pagination in multiple threads to not block on long searches
 	results, err := certDriver.QueryDomain(domainNode.Domain)
@@ -340,6 +345,21 @@ func visit(domainNode *graph.DomainNode) {
 	//  when we process the related domains
 }
 
+func printNode(domainNode *graph.DomainNode) {
+	if config.details {
+		fmt.Fprintln(os.Stdout, domainNode)
+	} else {
+		fmt.Fprintln(os.Stdout, domainNode.Domain)
+	}
+	if config.checkDNS && !domainNode.HasDNS {
+		// TODO print this in a better way
+		// TODO for debugging
+		realDomain, _ := dns.TLDPlus1(domainNode.Domain)
+		fmt.Fprintf(os.Stdout, "* Missing DNS for: %s\n", realDomain)
+
+	}
+}
+
 // certNodeFromCertResult convert certResult to certNode
 func certNodeFromCertResult(certResult *driver.CertResult) *graph.CertNode {
 	certNode := &graph.CertNode{

dns/ns.go

@@ -0,0 +1,87 @@
+package dns
+
+import (
+	"context"
+	"net"
+	"time"
+)
+
+var (
+	dnsCache    = make(map[string]bool)
+	dnsResolver = &net.Resolver{}
+)
+
+func init() {
+	//dnsResolver.PreferGo = true
+	dnsResolver.StrictErrors = false
+}
+
+func noSuchHostDNSError(err error) bool {
+	dnsErr, ok := err.(*net.DNSError)
+	if !ok {
+		// not a DNSError
+		return false
+	}
+	return dnsErr.Err == "no such host"
+}
+
+// HasRecords does NS, CNAME, A, and AAAA lookups with a timeout
+// returns error when no NS found, does not use TLDPlus1
+func HasRecords(domain string, timeout time.Duration) (bool, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	// first check for NS
+	ns, err := dnsResolver.LookupNS(ctx, domain)
+	if err != nil && !noSuchHostDNSError(err) {
+		//fmt.Println("NS error ", err)
+		return false, err
+	}
+	if len(ns) > 0 {
+		//fmt.Printf("Found %d NS for %s\n", len(ns), domain)
+		return true, nil
+	}
+
+	// next check for CNAME
+	cname, err := dnsResolver.LookupCNAME(ctx, domain)
+	if err != nil && !noSuchHostDNSError(err) {
+		//fmt.Println("cname error ", err)
+		return false, err
+	}
+	if len(cname) > 2 {
+		//fmt.Printf("found CNAME %s for %s\n", cname, domain)
+		return true, nil
+	}
+
+	// next check for IP
+	addrs, err := dnsResolver.LookupHost(ctx, domain)
+	if err != nil && !noSuchHostDNSError(err) {
+		//fmt.Println("ip error ", err)
+		return false, err
+	}
+	if len(addrs) > 0 {
+		//fmt.Printf("Found %d IPs for %s\n", len(addrs), domain)
+		return true, nil
+	}
+
+	//fmt.Printf("Found no DNS records for %s\n", domain)
+	return false, nil
+}
+
+// HasRecordsCache returns true if the domain has no DNS records (at the tldplus1 level)
+// uses a cache to store results to prevent lots of DNS lookups
+func HasRecordsCache(domain string, timeout time.Duration) (bool, error) {
+	domain, err := TLDPlus1(domain)
+	if err != nil {
+		return false, err
+	}
+	hasDNS, found := dnsCache[domain]
+	if found {
+		return hasDNS, nil
+	}
+	hasRecords, err := HasRecords(domain, timeout)
+	if err != nil {
+		dnsCache[domain] = hasRecords
+	}
+	return hasRecords, err
+}

dns/publicsuffix.go

@@ -0,0 +1,42 @@
+package dns
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/weppos/publicsuffix-go/publicsuffix"
+)
+
+var (
+	suffixListFindOptions = &publicsuffix.FindOptions{
+		IgnorePrivate: true,
+		DefaultRule:   publicsuffix.DefaultRule,
+	}
+	suffixListURL = "https://publicsuffix.org/list/public_suffix_list.dat"
+	suffixList    = publicsuffix.DefaultList
+	nsCache       = make(map[string]bool)
+)
+
+// UpdatePublicSuffixList gets a new copy of the public suffix list from the internat and updates the built in copy with the new rules
+func UpdatePublicSuffixList(timeout time.Duration) error {
+	suffixListParseOptions := &publicsuffix.ParserOption{
+		PrivateDomains: !suffixListFindOptions.IgnorePrivate,
+	}
+	client := http.Client{
+		Timeout: timeout,
+	}
+	resp, err := client.Get(suffixListURL)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	newSuffixList := publicsuffix.NewList()
+	newSuffixList.Load(resp.Body, suffixListParseOptions)
+	suffixList = newSuffixList
+	return err
+}
+
+// TLDPlus1 returns TLD+1 of domain
+func TLDPlus1(domain string) (string, error) {
+	return publicsuffix.DomainFromListWithOptions(suffixList, domain, suffixListFindOptions)
+}

driver/driver.go

@@ -47,7 +47,7 @@ type Result interface {
 // FingerprintMap stores a mapping of domains to Fingerprints returned from the driver
 // in the case where multiple domains where queries (redirects, related, etc..) the
 // matching certificates will be in this map
-// the fingerprints returned are guaranted to be a complete result for the domain's certs, but related domains may or may not be complete
+// the fingerprints returned are guaranteed to be a complete result for the domain's certs, but related domains may or may not be complete
 type FingerprintMap map[string][]fingerprint.Fingerprint
 
 // Add adds a domain and fingerprint to the map

go.mod

@@ -3,5 +3,6 @@ module github.com/lanrat/certgraph
 require (
 	github.com/lib/pq v1.0.0
 	github.com/weppos/publicsuffix-go v0.4.0
-	golang.org/x/net v0.0.0-20180826012351-8a410e7b638d
+	golang.org/x/net v0.0.0-20180906233101-161cd47e91fd // indirect
+	golang.org/x/text v0.3.0 // indirect
 )

graph/cert_node.go

@@ -2,11 +2,10 @@ package graph
 
 import (
 	"fmt"
-	"regexp"
 	"strings"
 
+	"github.com/lanrat/certgraph/dns"
 	"github.com/lanrat/certgraph/fingerprint"
-	"github.com/lanrat/certgraph/status"
 )
 
 // CertNode graph node to store certificate information
@@ -38,18 +37,22 @@ func (c *CertNode) AddFound(driver string) {
 }
 
 // CDNCert returns true if we think the certificate belongs to a CDN
-// very weak detection, only supports fastly & cloudflair
+// very weak detection, only supports fastly & cloudflare
 func (c *CertNode) CDNCert() bool {
 	for _, domain := range c.Domains {
-		// cloudflair
-		matched, _ := regexp.MatchString("([0-9][a-z])*\\.cloudflaressl\\.com", domain)
-		if matched {
+		// cloudflare
+		if strings.HasSuffix(domain, ".cloudflaressl.com") {
 			return true
 		}
 		// fastly
 		if strings.HasSuffix(domain, "fastly.net") {
 			return true
 		}
+		// akamai
+		if strings.HasSuffix(domain, ".akamai.net") {
+			return true
+		}
+
 	}
 	return false
 }
@@ -58,7 +61,7 @@ func (c *CertNode) CDNCert() bool {
 func (c *CertNode) TLDPlus1Count() int {
 	tldPlus1Domains := make(map[string]bool)
 	for _, domain := range c.Domains {
-		tldPlus1, err := status.TLDPlus1(domain)
+		tldPlus1, err := dns.TLDPlus1(domain)
 		if err != nil {
 			continue
 		}

graph/domain_node.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/lanrat/certgraph/dns"
 	"github.com/lanrat/certgraph/fingerprint"
 	"github.com/lanrat/certgraph/status"
 )
@@ -17,6 +19,7 @@ type DomainNode struct {
 	RelatedDomains status.Map
 	Status         status.Status
 	Root           bool
+	HasDNS         bool
 }
 
 // NewDomainNode constructor for DomainNode, converts domain to nonWildcard
@@ -41,6 +44,15 @@ func (d *DomainNode) AddRelatedDomains(domains []string) {
 	}
 }
 
+// CheckForDNS checks for the existence of DNS records for the domain's tld+1
+// sets the value to the node and returns the result as well
+func (d *DomainNode) CheckForDNS(timeout time.Duration) (bool, error) {
+	hasDNS, err := dns.HasRecordsCache(d.Domain, timeout)
+
+	d.HasDNS = hasDNS
+	return hasDNS, err
+}
+
 // AddStatusMap adds the status' in the map to the DomainNode
 // also sets the Node's own status if it is in the Map
 // side effect: will delete its own status from the provided map
@@ -94,5 +106,6 @@ func (d *DomainNode) ToMap() map[string]string {
 	m["root"] = strconv.FormatBool(d.Root)
 	m["depth"] = strconv.FormatUint(uint64(d.Depth), 10)
 	m["related"] = relatedString
+	m["hasDNS"] = strconv.FormatBool(d.HasDNS)
 	return m
 }