Merge pull request #113 from hubblestack/develop

Merge to master (prep v2017.9.0)

Author: basepi

README.md

@@ -59,7 +59,7 @@ gitfs_remotes:
   - https://github.com/hubblestack/hubblestack_data.git:
     - root: ''
   - https://github.com/hubblestack/hubble-salt.git:
-    - base: v2017.8.3
+    - base: v2017.9.0
     - root: ''
 ```
 

_beacons/pulsar.py

@@ -39,7 +39,7 @@
     DEFAULT_MASK = None
 
 __virtualname__ = 'pulsar'
-__version__ = 'v2017.8.3'
+__version__ = 'v2017.9.0'
 CONFIG = None
 CONFIG_STALENESS = 0
 

_beacons/win_pulsar.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+import salt.utils
+import salt.modules.cmdmod
+
+__salt__ = { 'cmd.run': salt.modules.cmdmod._run_quiet }
+
+def osquerygrain():
+    '''
+    Return osquery version in grain
+    '''
+    # Provides:
+    #   osqueryversion
+    #   osquerybinpath
+    grains = {}
+    option = '--version'
+
+    # Prefer our /opt/osquery/osqueryi if present
+    osqueryipaths = ('/opt/osquery/osqueryi', 'osqueryi', '/usr/bin/osqueryi')
+    for path in osqueryipaths:
+        if salt.utils.which(path):
+            for item in __salt__['cmd.run']('{0} {1}'.format(path, option)).split():
+                if item[:1].isdigit():
+                    grains['osqueryversion'] = item
+                    grains['osquerybinpath'] = salt.utils.which(path)
+                    break
+            break
+    return grains

_grains/osqueryinfo.py

@@ -35,7 +35,7 @@
 from nova_loader import NovaLazyLoader
 
 __nova__ = {}
-__version__ = 'v2017.8.3'
+__version__ = 'v2017.9.0'
 
 
 def audit(configs=None,
@@ -112,7 +112,7 @@ def audit(configs=None,
                    show_success=show_success,
                    show_compliance=show_compliance)
 
-    if __salt__['config.get']('hubblestack:nova:autoload', True):
+    if not called_from_top and __salt__['config.get']('hubblestack:nova:autoload', True):
         load()
     if not __nova__:
         return False, 'No nova modules/data have been loaded.'
@@ -465,9 +465,14 @@ def top(topfile='top.nova',
         else:
             if 'Errors' not in results:
                 results['Errors'] = {}
-            results['Errors'][topfile] = {'error': 'topfile malformed, list '
-                                                   'entries must be strings or dicts'}
-            return results
+            error_log = 'topfile malformed, list entries must be strings or '\
+                        'dicts: {0}'.format(data)
+            results['Errors'][topfile] = {'error': error_log}
+            log.error(error_log)
+            continue
+
+    if not data_by_tag:
+        return results
 
     # Run the audits
     for tag, data in data_by_tag.iteritems():

_modules/hubble.py

@@ -34,13 +34,14 @@
 import os
 import sys
 import yaml
+import collections
 
 import salt.utils
 from salt.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 
-__version__ = 'v2017.8.3'
+__version__ = 'v2017.9.0'
 __virtualname__ = 'nebula'
 
 
@@ -74,13 +75,34 @@ def queries(query_group,
         salt '*' nebula.queries hour verbose=True
         salt '*' nebula.queries hour pillar_key=sec_osqueries
     '''
+    query_data = {}
     MAX_FILE_SIZE = 104857600
     if query_file is None:
         if salt.utils.is_windows():
             query_file = 'salt://hubblestack_nebula/hubblestack_nebula_win_queries.yaml'
         else:
             query_file = 'salt://hubblestack_nebula/hubblestack_nebula_queries.yaml'
-    if not salt.utils.which('osqueryi'):
+    if not isinstance(query_file, list):
+        query_file = [query_file]
+    for fh in query_file:
+        if 'salt://' in fh:
+            orig_fh = fh
+            fh = __salt__['cp.cache_file'](fh)
+        if fh is None:
+            log.error('Could not find file {0}.'.format(orig_fh))
+            return None
+        if os.path.isfile(fh):
+            with open(fh, 'r') as f:
+                f_data = yaml.safe_load(f)
+                if not isinstance(f_data, dict):
+                    raise CommandExecutionError('File data is not formed as a dict {0}'
+                                                .format(f_data))
+                query_data =  _dict_update(query_data,
+                                           f_data,
+                                           recursive_update=True,
+                                           merge_lists=True)
+
+    if 'osquerybinpath' not in __grains__:
         if query_group == 'day':
             log.warning('osquery not installed on this host. Returning baseline data')
             # Match the formatting of normal osquery results. Not super
@@ -140,19 +162,6 @@ def queries(query_group,
             else:
                return None
 
-
-    orig_filename = query_file
-    query_file = __salt__['cp.cache_file'](query_file)
-    if query_file is None:
-        log.error('Could not find file {0}.'.format(orig_filename))
-        return None
-    with open(query_file, 'r') as fh:
-        query_data = yaml.safe_load(fh)
-
-    if not isinstance(query_data, dict):
-        raise CommandExecutionError('Query data is not formed as a dict {0}'
-                                    .format(query_data))
-
     query_data = query_data.get(query_group, [])
 
     if not query_data:
@@ -170,7 +179,7 @@ def queries(query_group,
             'result': True,
         }
 
-        cmd = ['osqueryi', '--read_max', MAX_FILE_SIZE, '--json', query_sql]
+        cmd = [__grains__['osquerybinpath'], '--read_max', MAX_FILE_SIZE, '--json', query_sql]
         res = __salt__['cmd.run_all'](cmd)
         if res['retcode'] == 0:
             query_ret['data'] = json.loads(res['stdout'])
@@ -192,7 +201,7 @@ def queries(query_group,
         for query_name, query_ret in r.iteritems():
             for result in query_ret['data']:
                 for key, value in result.iteritems():
-                    if value.startswith('__JSONIFY__'):
+                    if value and isinstance(value, str) and value.startswith('__JSONIFY__'):
                         result[key] = json.loads(value[len('__JSONIFY__'):])
 
     return ret
@@ -268,3 +277,97 @@ def hubble_versions():
 
     return {'hubble_versions': {'data': [versions],
                                 'result': True}}
+
+
+def top(query_group,
+        topfile='salt://hubblestack_nebula/top.nebula',
+        verbose=False,
+        report_version_with_day=True):
+
+    if salt.utils.is_windows():
+        topfile = 'salt://hubblestack_nebula/win_top.nebula'
+
+    configs = get_top_data(topfile)
+
+    configs = ['salt://hubblestack_nebula/' + config.replace('.', '/') + '.yaml'
+               for config in configs]
+
+    return queries(query_group,
+                   query_file=configs,
+                   verbose=False,
+                   report_version_with_day=True)
+
+
+def get_top_data(topfile):
+
+    topfile = __salt__['cp.cache_file'](topfile)
+
+    try:
+        with open(topfile) as handle:
+            topdata = yaml.safe_load(handle)
+    except Exception as e:
+        raise CommandExecutionError('Could not load topfile: {0}'.format(e))
+
+    if not isinstance(topdata, dict) or 'nebula' not in topdata or \
+            not(isinstance(topdata['nebula'], dict)):
+        raise CommandExecutionError('Nebula topfile not formatted correctly')
+
+    topdata = topdata['nebula']
+
+    ret = []
+
+    for match, data in topdata.iteritems():
+        if __salt__['match.compound'](match):
+            ret.extend(data)
+
+    return ret
+
+
+def _dict_update(dest, upd, recursive_update=True, merge_lists=False):
+    '''
+    Recursive version of the default dict.update
+
+    Merges upd recursively into dest
+
+    If recursive_update=False, will use the classic dict.update, or fall back
+    on a manual merge (helpful for non-dict types like FunctionWrapper)
+
+    If merge_lists=True, will aggregate list object types instead of replace.
+    This behavior is only activated when recursive_update=True. By default
+    merge_lists=False.
+    '''
+    if (not isinstance(dest, collections.Mapping)) \
+            or (not isinstance(upd, collections.Mapping)):
+        raise TypeError('Cannot update using non-dict types in dictupdate.update()')
+    updkeys = list(upd.keys())
+    if not set(list(dest.keys())) & set(updkeys):
+        recursive_update = False
+    if recursive_update:
+        for key in updkeys:
+            val = upd[key]
+            try:
+                dest_subkey = dest.get(key, None)
+            except AttributeError:
+                dest_subkey = None
+            if isinstance(dest_subkey, collections.Mapping) \
+                    and isinstance(val, collections.Mapping):
+                ret = update(dest_subkey, val, merge_lists=merge_lists)
+                dest[key] = ret
+            elif isinstance(dest_subkey, list) \
+                     and isinstance(val, list):
+                if merge_lists:
+                    dest[key] = dest.get(key, []) + val
+                else:
+                    dest[key] = upd[key]
+            else:
+                dest[key] = upd[key]
+        return dest
+    else:
+        try:
+            for k in upd.keys():
+                dest[k] = upd[k]
+        except AttributeError:
+            # this mapping is not a dict
+            for k in upd:
+                dest[k] = upd[k]
+        return dest

_modules/nebula_osquery.py

@@ -28,7 +28,7 @@
 CONFIG = None
 CONFIG_STALENESS = 0
 
-__version__ = 'v2017.8.3'
+__version__ = 'v2017.9.0'
 
 
 def __virtual__():
@@ -158,9 +158,10 @@ def process(configfile='salt://hubblestack_pulsar/hubblestack_pulsar_win_config.
     # Validate ACLs on watched folders/files and add if needed
     if update_acls:
         for path in config:
-            if path == 'win_notify_interval' or path == 'return' or path == 'batch' or path == 'checksum' or path == 'stats':
+            if path in ['win_notify_interval', 'return', 'batch', 'checksum', 'stats', 'paths', 'verbose']:
                 continue
             if not os.path.exists(path):
+                log.info('The folder path {0} does not exist'.format(path))
                 continue
             if isinstance(config[path], dict):
                 mask = config[path].get('mask', DEFAULT_MASK)
@@ -223,6 +224,21 @@ def process(configfile='salt://hubblestack_pulsar/hubblestack_pulsar_win_config.
     return ret
 
 
+def canary(change_file=None):
+    '''
+    Simple module to change a file to trigger a FIM event (daily, etc)
+
+    THE SPECIFIED FILE WILL BE CREATED AND DELETED
+
+    Defaults to CONF_DIR/fim_canary.tmp, i.e. /etc/hubble/fim_canary.tmp
+    '''
+    if change_file is None:
+        conf_dir = os.path.dirname(__opts__['conf_file'])
+        change_file = os.path.join(conf_dir, 'fim_canary.tmp')
+    __salt__['file.touch'](change_file)
+    __salt__['file.remove'](change_file)
+
+
 def _check_acl(path, mask, wtype, recurse):
     audit_dict = {}
     success = True
@@ -551,3 +567,39 @@ def _dict_update(dest, upd, recursive_update=True, merge_lists=False):
             for k in upd:
                 dest[k] = upd[k]
         return dest
+
+
+def top(topfile='salt://hubblestack_pulsar/win_top.pulsar',
+        verbose=False):
+
+    configs = get_top_data(topfile)
+
+    configs = ['salt://hubblestack_pulsar/' + config.replace('.','/') + '.yaml'
+               for config in configs]
+
+    return process(configs, verbose=verbose)
+
+
+def get_top_data(topfile):
+
+    topfile = __salt__['cp.cache_file'](topfile)
+
+    try:
+        with open(topfile) as handle:
+            topdata = yaml.safe_load(handle)
+    except Exception as e:
+        raise CommandExecutionError('Could not load topfile: {0}'.format(e))
+
+    if not isinstance(topdata, dict) or 'pulsar' not in topdata or \
+            not(isinstance(topdata['pulsar'], dict)):
+        raise CommandExecutionError('Pulsar topfile not formatted correctly')
+
+    topdata = topdata['pulsar']
+
+    ret = []
+
+    for match, data in topdata.iteritems():
+        if __salt__['match.compound'](match):
+            ret.extend(data)
+
+    return ret

_modules/win_pulsar.py

@@ -69,7 +69,7 @@
 # Import Salt Libs
 import salt.returners
 
-__version__ = 'v2017.8.3'
+__version__ = 'v2017.9.0'
 
 log = logging.getLogger(__name__)