probing.yaml

name: probing
desc: Examining the DNS and HTTP host of the target

report:
  final:
    - "{{Output}}/probing/dns-{{Workspace}}.txt"
    - "{{Output}}/probing/http-{{Workspace}}.txt"
    - "{{Output}}/probing/new-dns-{{Workspace}}.txt"

pre_run:
  - CreateFolder("{{Output}}/probing")
  - CreateFolder("{{Output}}/ipspace")

params:
  - finalSubdomainFile: "{{Output}}/subdomain/final-{{Workspace}}.txt"
  - permDomainFile: "{{Output}}/probing/permutation-{{Workspace}}.txt"
  - rawDnsFile: "{{Output}}/probing/raw-{{Workspace}}.txt"
  - customDnsWordlist: "{{Output}}/probing/customDnsWordlist-{{Workspace}}.txt"
  - permWordlists: "{{Data}}/wordlists/dns/altdns.txt"
  - bruteWordlist: "{{Data}}/wordlists/dns/medium.txt"
  - resolvers: "{{Data}}/mics/resolvers.txt"
  - trustedResolvers: "{{Data}}/mics/trusted-resolvers.txt"
  - permLimit: "8000"
  - wildcardLimit: "100000" # if the httpFile is too big, we need to filter it as it might be a wildcard domains
  - enablePermutation: "false"
  - enableDnsBruteFocing: "false"
  - dnsJsonFile: "{{Output}}/probing/dns-json-{{Workspace}}.txt"
  - dnsFile: "{{Output}}/probing/dns-{{Workspace}}.txt"
  - httpFile: "{{Output}}/probing/http-{{Workspace}}.txt"
  - massdnsRateBrute: "{{ threads * 50 }}"
  - dnsThreads: "{{ threads * 25 }}"
  - httpThreads: "{{ threads * 8 }}"
  - dnsxThreads: "{{ threads * 8 }}"
  - wildcardTests: "{{ threads * 3 }}"
  - defaultUA: "User-Agent: Mozilla/5.0 (compatible; Osmedeus/v4; +https://github.com/j3ssie/osmedeus)"

steps:
  # check if we already scan before
  - conditions:
      - "FileLength('{{dnsFile}}') > 1"
    commands:
      - cp {{dnsFile}} {{Output}}/probing/bak-dns-{{Workspace}}.txt
      - cp {{httpFile}} {{Output}}/probing/bak-http-{{Workspace}}.txt
    scripts:
      # joining the old subdomains with a new one too
      - ExecCmd("cat {{Output}}/probing/subbrute-{{Workspace}}.txt >> {{finalSubdomainFile}}")
      - SortU("{{finalSubdomainFile}}")

  ##### DNS Brute Forcing
  # build the custom wordlists based on the subdomain file
  - commands:
      - "cat {{finalSubdomainFile}} | {{Binaries}}/str-replace -d '.' -n | sort -u >> {{customDnsWordlist}}"
      - 'cat {{bruteWordlist}} >> {{customDnsWordlist}}'
    scripts:
      - "SortU('{{customDnsWordlist}}')"
  # simple bruteforce with main domain
  - conditions:
      - '"{{enableDnsBruteFocing}}" == "true"'
    required:
      - "{{Binaries}}/puredns"
      - "{{Binaries}}/massdns"
    commands:
      - '{{Binaries}}/puredns bruteforce {{customDnsWordlist}} {{Target}} -r {{resolvers}} --wildcard-tests {{wildcardTests}} -w {{Output}}/probing/subbrute-{{Workspace}}.txt --resolvers-trusted {{trustedResolvers}} --rate-limit-trusted {{massdnsRateBrute}} --bin {{Binaries}}/massdns >/dev/null 2>&1'
    scripts:
      - ExecCmd("cat {{Output}}/probing/subbrute-{{Workspace}}.txt >> {{finalSubdomainFile}}")
      - SortU("{{finalSubdomainFile}}")

  # generate permutation DNS when there is not too much domains
  - conditions:
      - "FileLength('{{finalSubdomainFile}}') <= {{permLimit}}"
      - '"{{enablePermutation}}" == "true"'
      - '"{{enableDnsBruteFocing}}" == "true"'
    scripts:
      - Printf("Generating permutation subdomains wordlists which might create a very big file")
      - ExecCmd("cat {{finalSubdomainFile}} | {{Binaries}}/alterx -silent >> {{permDomainFile}}")
      - SortU('{{permDomainFile}}')
  ##### end of DNS Brute Forcing

  - commands:
      - "cat {{finalSubdomainFile}} {{permDomainFile}} > {{rawDnsFile}}"
  # actually resolving all the DNS
  - required:
      - "{{Binaries}}/puredns"
      - "{{Binaries}}/massdns"
      - "{{rawDnsFile}}"
    commands:
      - "{{Binaries}}/puredns resolve {{rawDnsFile}} -r {{resolvers}} --write {{Output}}/probing/nonwild-{{Workspace}}.txt --write-massdns {{dnsFile}} --write-wildcards {{Output}}/probing/wildcards-{{Workspace}}.txt --wildcard-tests {{wildcardTests}} --resolvers-trusted {{trustedResolvers}} --rate-limit-trusted {{massdnsRateBrute}} --bin {{Binaries}}/massdns >/dev/null 2>&1"
    scripts:
      - "SortU('{{dnsFile}}')"

  # in case we have no DNS result
  - conditions:
      - "FileLength('{{dnsFile}}') < 1"
    commands:
      - "cat {{rawDnsFile}} | {{Binaries}}/dnsx -silent -a -cname -resp-only -json -r {{resolvers}} -t {{httpThreads}} > {{Output}}/probing/raw-dnsx-{{Workspace}}.txt"
    scripts:
      - "CleanJSONDnsx('raw-dnsx-{{Workspace}}.txt', '{{dnsFile}}')"

  # probing for http
  - required:
      - "{{Binaries}}/httprobe"
    commands:
      - "cat {{rawDnsFile}} {{finalSubdomainFile}} | sort -u | {{Binaries}}/httpx -H '{{defaultUA}}' -silent -t {{httpThreads}} >> {{httpFile}}"
    scripts:
      - "SortU('{{httpFile}}')"
      - "ExecCmd('rm -rf {{rawDnsFile}} {{customDnsWordlist}} {{permDomainFile}}')"

  # if the httpFile is too big, we need to filter it as it might be a wildcard domains
  - conditions:
      - "FileLength('{{httpFile}}') > {{wildcardLimit}}"
    scripts:
      - Warnf('HTTP File {{httpFile}} is too big and excess the limit {{wildcardLimit}}. Filtering it...')
      - ExecCmd("rm -rf {{httpFile}}")
      - ExecCmd("cat {{dnsFile}} | grep '{{Target}}' | awk '{print $1}' | sed 's/\.$//g' | sort -u | {{Binaries}}/httprobe -c {{httpThreads}} >> {{httpFile}}")

  # get IP Address for later modules
  - required:
      - '{{dnsFile}}'
    commands:
      - cat '{{dnsFile}}' | grep ' A ' | awk '{print $3}' | sort -u >> {{Output}}/ipspace/{{Workspace}}-ip.txt
    scripts:
      - SortU("{{Output}}/ipspace/{{Workspace}}-ip.txt")

  ### Checking if we have new HTTP or DNS results
  # generate new dns 
  - conditions:
      - "FileLength('{{Output}}/probing/bak-dns-{{Workspace}}.txt') > 1"
    scripts:
      - ExecCmd("cat {{dnsFile}} | anew {{Output}}/probing/bak-dns-{{Workspace}}.txt > {{Output}}/probing/new-dns-{{Workspace}}.txt")
      - SortU("{{Output}}/probing/new-dns-{{Workspace}}.txt")
      - "Printf('New DNS detected: {{Output}}/probing/new-dns-{{Workspace}}.txt')"
  # generate new http 
  - conditions:
      - "FileLength('{{Output}}/probing/bak-http-{{Workspace}}.txt') > 1"
    scripts:
      - ExecCmd("cat {{httpFile}} | anew {{Output}}/probing/bak-http-{{Workspace}}.txt > {{Output}}/probing/new-http-{{Workspace}}.txt")
      - SortU("{{Output}}/probing/new-http-{{Workspace}}.txt")
      - "Printf('New HTTP detected: {{Output}}/probing/new-http-{{Workspace}}.txt')"

  - required:
      - "{{httpFile}}"
    scripts:
      - TotalSubdomain('{{httpFile}}')
      - TotalDns('{{dnsFile}}')