Skip to content

Commit 6a89ca8

Browse files
basepibasepi
authored
Merge pull request #107 from hubblestack/develop
Merge to master (prep v2017.8.3)
2 parents 0fca004 + 50a2fb2 commit 6a89ca8

File tree

69 files changed

+485
-238128
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+485
-238128
lines changed

‎README.md

+5-2
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ https://hubblestack.io
4242

4343
This installation method subscribes directly to our GitHub repository, pinning
4444
to a tag or branch. This method requires no package installation or manual
45-
checkouts.
45+
checkouts. It also will subscribe to our policy repo,
46+
[hubblestack_data](https://github.com/hubblestack/hubblestack_data).
4647

4748
Requirements: GitFS support on your Salt Master. (Usually just requires
4849
installation of `gitpython` or `pygit2`. `pygit2` is the recommended gitfs
@@ -55,8 +56,10 @@ fileserver_backend:
5556
- roots
5657
- git
5758
gitfs_remotes:
59+
- https://github.com/hubblestack/hubblestack_data.git:
60+
- root: ''
5861
- https://github.com/hubblestack/hubble-salt.git:
59-
- base: v2017.8.2
62+
- base: v2017.8.3
6063
- root: ''
6164
```
6265

‎_beacons/pulsar.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
DEFAULT_MASK = None
4040

4141
__virtualname__ = 'pulsar'
42-
__version__ = 'v2017.8.2'
42+
__version__ = 'v2017.8.3'
4343
CONFIG = None
4444
CONFIG_STALENESS = 0
4545

‎_beacons/win_pulsar.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
DEFAULT_TYPE = 'all'
2727

2828
__virtualname__ = 'pulsar'
29-
__version__ = 'v2017.8.2'
29+
__version__ = 'v2017.8.3'
3030
CONFIG = None
3131
CONFIG_STALENESS = 0
3232

‎_modules/hubble.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
from nova_loader import NovaLazyLoader
3636

3737
__nova__ = {}
38-
__version__ = 'v2017.8.2'
38+
__version__ = 'v2017.8.3'
3939

4040

4141
def audit(configs=None,

‎_modules/nebula_osquery.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040

4141
log = logging.getLogger(__name__)
4242

43-
__version__ = 'v2017.8.2'
43+
__version__ = 'v2017.8.3'
4444
__virtualname__ = 'nebula'
4545

4646

‎_modules/win_pulsar.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
CONFIG = None
2929
CONFIG_STALENESS = 0
3030

31-
__version__ = 'v2017.8.2'
31+
__version__ = 'v2017.8.3'
3232

3333

3434
def __virtual__():

‎_returners/slack_pulsar_returner.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
# Import Salt Libs
7070
import salt.returners
7171

72-
__version__ = 'v2017.8.2'
72+
__version__ = 'v2017.8.3'
7373

7474
log = logging.getLogger(__name__)
7575

‎_returners/splunk_nebula_return.py

+103-99
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050

5151
import logging
5252

53-
__version__ = 'v2017.8.2'
53+
__version__ = 'v2017.8.3'
5454

5555
_max_content_bytes = 100000
5656
http_event_collector_SSL_verify = False
@@ -62,106 +62,109 @@
6262

6363

6464
def returner(ret):
65-
opts_list = _get_options()
66-
67-
# Get cloud details
68-
clouds = get_cloud_details()
69-
70-
for opts in opts_list:
71-
logging.info('Options: %s' % json.dumps(opts))
72-
http_event_collector_key = opts['token']
73-
http_event_collector_host = opts['indexer']
74-
http_event_collector_port = opts['port']
75-
hec_ssl = opts['http_event_server_ssl']
76-
proxy = opts['proxy']
77-
timeout = opts['timeout']
78-
custom_fields = opts['custom_fields']
79-
80-
# Set up the fields to be extracted at index time. The field values must be strings.
81-
# Note that these fields will also still be available in the event data
82-
index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId']
83-
try:
84-
index_extracted_fields.extend(opts['index_extracted_fields'])
85-
except TypeError:
86-
pass
87-
88-
# Set up the collector
89-
hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_port=http_event_collector_port, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)
90-
91-
# st = 'salt:hubble:nova'
92-
data = ret['return']
93-
minion_id = ret['id']
94-
jid = ret['jid']
95-
master = __grains__['master']
96-
fqdn = __grains__['fqdn']
97-
# Sometimes fqdn is blank. If it is, replace it with minion_id
98-
fqdn = fqdn if fqdn else minion_id
99-
try:
100-
fqdn_ip4 = __grains__['fqdn_ip4'][0]
101-
except IndexError:
102-
fqdn_ip4 = __grains__['ipv4'][0]
103-
if fqdn_ip4.startswith('127.'):
104-
for ip4_addr in __grains__['ipv4']:
105-
if ip4_addr and not ip4_addr.startswith('127.'):
106-
fqdn_ip4 = ip4_addr
107-
break
108-
109-
if not data:
110-
return
111-
else:
112-
for query in data:
113-
for query_name, query_results in query.iteritems():
114-
for query_result in query_results['data']:
115-
event = {}
116-
payload = {}
117-
event.update(query_result)
118-
event.update({'query': query_name})
119-
event.update({'job_id': jid})
120-
event.update({'master': master})
121-
event.update({'minion_id': minion_id})
122-
event.update({'dest_host': fqdn})
123-
event.update({'dest_ip': fqdn_ip4})
124-
125-
for cloud in clouds:
126-
event.update(cloud)
127-
128-
for custom_field in custom_fields:
129-
custom_field_name = 'custom_' + custom_field
130-
custom_field_value = __salt__['config.get'](custom_field, '')
131-
if isinstance(custom_field_value, str):
132-
event.update({custom_field_name: custom_field_value})
133-
elif isinstance(custom_field_value, list):
134-
custom_field_value = ','.join(custom_field_value)
135-
event.update({custom_field_name: custom_field_value})
136-
137-
payload.update({'host': fqdn})
138-
payload.update({'index': opts['index']})
139-
if opts['add_query_to_sourcetype']:
140-
payload.update({'sourcetype': "%s_%s" % (opts['sourcetype'], query_name)})
141-
else:
142-
payload.update({'sourcetype': opts['sourcetype']})
143-
payload.update({'event': event})
144-
145-
# Potentially add metadata fields:
146-
fields = {}
147-
for item in index_extracted_fields:
148-
if item in payload['event'] and not isinstance(payload['event'][item], (list, dict, tuple)):
149-
fields[item] = str(payload['event'][item])
150-
if fields:
151-
payload.update({'fields': fields})
152-
153-
# If the osquery query includes a field called 'time' it will be checked.
154-
# If it's within the last year, it will be used as the eventtime.
155-
event_time = query_result.get('time', '')
156-
try:
157-
if (datetime.fromtimestamp(time.time()) - datetime.fromtimestamp(float(event_time))).days > 365:
65+
try:
66+
opts_list = _get_options()
67+
68+
# Get cloud details
69+
clouds = get_cloud_details()
70+
71+
for opts in opts_list:
72+
logging.info('Options: %s' % json.dumps(opts))
73+
http_event_collector_key = opts['token']
74+
http_event_collector_host = opts['indexer']
75+
http_event_collector_port = opts['port']
76+
hec_ssl = opts['http_event_server_ssl']
77+
proxy = opts['proxy']
78+
timeout = opts['timeout']
79+
custom_fields = opts['custom_fields']
80+
81+
# Set up the fields to be extracted at index time. The field values must be strings.
82+
# Note that these fields will also still be available in the event data
83+
index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId']
84+
try:
85+
index_extracted_fields.extend(opts['index_extracted_fields'])
86+
except TypeError:
87+
pass
88+
89+
# Set up the collector
90+
hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_port=http_event_collector_port, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)
91+
92+
# st = 'salt:hubble:nova'
93+
data = ret['return']
94+
minion_id = ret['id']
95+
jid = ret['jid']
96+
master = __grains__['master']
97+
fqdn = __grains__['fqdn']
98+
# Sometimes fqdn is blank. If it is, replace it with minion_id
99+
fqdn = fqdn if fqdn else minion_id
100+
try:
101+
fqdn_ip4 = __grains__['fqdn_ip4'][0]
102+
except IndexError:
103+
fqdn_ip4 = __grains__['ipv4'][0]
104+
if fqdn_ip4.startswith('127.'):
105+
for ip4_addr in __grains__['ipv4']:
106+
if ip4_addr and not ip4_addr.startswith('127.'):
107+
fqdn_ip4 = ip4_addr
108+
break
109+
110+
if not data:
111+
return
112+
else:
113+
for query in data:
114+
for query_name, query_results in query.iteritems():
115+
for query_result in query_results['data']:
116+
event = {}
117+
payload = {}
118+
event.update(query_result)
119+
event.update({'query': query_name})
120+
event.update({'job_id': jid})
121+
event.update({'master': master})
122+
event.update({'minion_id': minion_id})
123+
event.update({'dest_host': fqdn})
124+
event.update({'dest_ip': fqdn_ip4})
125+
126+
for cloud in clouds:
127+
event.update(cloud)
128+
129+
for custom_field in custom_fields:
130+
custom_field_name = 'custom_' + custom_field
131+
custom_field_value = __salt__['config.get'](custom_field, '')
132+
if isinstance(custom_field_value, str):
133+
event.update({custom_field_name: custom_field_value})
134+
elif isinstance(custom_field_value, list):
135+
custom_field_value = ','.join(custom_field_value)
136+
event.update({custom_field_name: custom_field_value})
137+
138+
payload.update({'host': fqdn})
139+
payload.update({'index': opts['index']})
140+
if opts['add_query_to_sourcetype']:
141+
payload.update({'sourcetype': "%s_%s" % (opts['sourcetype'], query_name)})
142+
else:
143+
payload.update({'sourcetype': opts['sourcetype']})
144+
payload.update({'event': event})
145+
146+
# Potentially add metadata fields:
147+
fields = {}
148+
for item in index_extracted_fields:
149+
if item in payload['event'] and not isinstance(payload['event'][item], (list, dict, tuple)):
150+
fields[item] = str(payload['event'][item])
151+
if fields:
152+
payload.update({'fields': fields})
153+
154+
# If the osquery query includes a field called 'time' it will be checked.
155+
# If it's within the last year, it will be used as the eventtime.
156+
event_time = query_result.get('time', '')
157+
try:
158+
if (datetime.fromtimestamp(time.time()) - datetime.fromtimestamp(float(event_time))).days > 365:
159+
event_time = ''
160+
except:
158161
event_time = ''
159-
except:
160-
event_time = ''
161-
finally:
162-
hec.batchEvent(payload, eventtime=event_time)
162+
finally:
163+
hec.batchEvent(payload, eventtime=event_time)
163164

164-
hec.flushBatch()
165+
hec.flushBatch()
166+
except:
167+
log.exception('Error ocurred in splunk_nebula_return')
165168
return
166169

167170

@@ -202,6 +205,7 @@ def _get_options():
202205
splunk_opts['proxy'] = __salt__['config.get']('hubblestack:nebula:returner:splunk:proxy', {})
203206
splunk_opts['timeout'] = __salt__['config.get']('hubblestack:nebula:returner:splunk:timeout', 9.05)
204207
splunk_opts['index_extracted_fields'] = __salt__['config.get']('hubblestack:nebula:returner:splunk:index_extracted_fields', [])
208+
splunk_opts['port'] = __salt__['config.get']('hubblestack:nebula:returner:splunk:port', '8088')
205209

206210
return [splunk_opts]
207211

0 commit comments

Comments
 (0)