pistar-watchdog

#!/usr/bin/python

###############################################################################
#                                                                             #
#                         Pi-Star Service Watchdog                            #
#                                                                             #
#    Version 2.0, Code, Design and Development by Andy Taylor (MW0MWZ).       #
#                                                                             #
#   The service watchdog will monitor the D-Star Services, restarting them    #
#   after any kind of failures - something that used to happen occasionally   #
#             when using the origional G4KLX D-Star software.                 #
#                                                                             #
###############################################################################

import subprocess
import time
import os
import linecache
import datetime

import sys					#
if (sys.version_info.major > 2):		#
	import configparser as ConfigParser	# python 3
else:						#
	import ConfigParser			# python 2


time.sleep(90) #Assumiing the Pi has just started up, let the services settle...

# Get the disk usage
def disk_stat(path):
    disk = os.statvfs(path)
    percent = (disk.f_blocks - disk.f_bfree) * 100 / (disk.f_blocks - disk.f_bfree + disk.f_bavail) + 1
    return percent

while True: #Main loop
	# Get the current status
	dstarrepeater = "/etc/dstar-radio.dstarrepeater"
	mmdvmhost = "/etc/dstar-radio.mmdvmhost"
	if os.stat(mmdvmhost).st_size == 0: #
		mmdvmhost = "/etc/dstar-radio.mmdvmhost-disabled" #

	# Check the disk usage
	if disk_stat('/var/log') > 90:
		# Clean up the log volume
		os.system('rm -rf /var/log/*.1.gz')
		os.system('rm -rf /var/log/*.1')
		os.system('echo "$(tail -500 /var/log/nginx/error.log)" > /var/log/nginx/error.log')
		os.system('rm -f $(find /var/log/pi-star/*.log -type f -mtime +0 -print)')
		os.system('rm -f $(find /var/log/ -type f -mtime +0 -print | grep .gz)')
		time.sleep(60) # Give the service secs before we re-test
	if disk_stat('/var/log') > 90:
		# Shorten the Pi-Star logs
		utcnow = datetime.datetime.utcnow()
		datenow = utcnow.strftime('%Y-%m-%d')
		if os.path.isfile(dstarrepeater):
			dstarLogOld = '/var/log/pi-star/DStarRepeater-' + datenow + '.log'
			dstarLogNew = '/var/log/pi-star/dstarrepeaterd-' + datenow + '.log'
			if os.path.isfile(dstarLogOld):
				os.system('echo "$(tail -1500 /var/log/pi-star/DStarRepeater-$(date +%F).log)" > /var/log/pi-star/DStarRepeater-$(date +%F).log')
			if os.path.isfile(dstarLogNew):
				os.system('echo "$(tail -1500 /var/log/pi-star/dstarrepeaterd-$(date +%F).log)" > /var/log/pi-star/dstarrepeaterd-$(date +%F).log')
			os.system('/bin/systemctl restart dstarrepeater.service')
			time.sleep(60) # Give the service 60 secs before we re-test
	if disk_stat('/var/log') > 90:
		utcnow = datetime.datetime.utcnow()
		datenow = utcnow.strftime('%Y-%m-%d')
		if os.path.isfile(mmdvmhost):
			mmdvmLog = '/var/log/pi-star/MMDVM-' + datenow + '.log'
			if os.path.isfile(mmdvmLog):
				os.system('echo "$(tail -1500 /var/log/pi-star/MMDVM-$(date +%F).log)" > /var/log/pi-star/MMDVM-$(date +%F).log')
			os.system('/bin/systemctl restart mmdvmhost.service')
			time.sleep(60) # Give the service 60 secs before we re-test

	# Check the D-Star Specific Services
	if os.path.isfile(dstarrepeater):
		checkprocdstar = subprocess.Popen('pgrep' + ' dstarrepeaterd', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocdstar.stdout.readlines():
			os.system('/bin/systemctl stop dstarrepeater.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dstarrepeater.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdstar.wait()

		# At this point it's a safe bet that the service is running, now look for signs of DVAP failure.
		utcnow = datetime.datetime.utcnow()
		datenow = utcnow.strftime('%Y-%m-%d')
		dateminus300sec = datetime.datetime.utcnow() - datetime.timedelta(minutes=5)
		logstampnow = utcnow.strftime('%Y-%m-%d %H:%M:%S')
		logstampnowminus300sec = dateminus300sec.strftime('%Y-%m-%d %H:%M:%S')
		# Best figure out the log name now
		currentLogOld = '/var/log/pi-star/DStarRepeater-' + datenow + '.log'
		currentLogNew = '/var/log/pi-star/dstarrepeaterd-' + datenow + '.log'
		if os.path.isfile(currentLogOld):
			currentLog = currentLogOld
		if os.path.isfile(currentLogNew):
			currentLog = currentLogNew

		# Open the DStarRepeater Log
		logfile = open(currentLog, 'r')
		loglist = logfile.readlines()
		logfile.close()

		# Parse the log lines
		for line in loglist:
			# We only care about logs in the last 60 secs
			if line[3:22] >= logstampnowminus300sec and line[3:22] <= logstampnow:
				# Now we look for DVAP failure lines
				if (str('Bad DVAP header') in line) or (str('Cannot open the D-Star modem') in line):
					os.system('/bin/systemctl stop dstarrepeater.service')
					time.sleep(30) # Give the DVAP time to recover
					os.system('/bin/systemctl start dstarrepeater.service')
					time.sleep(270) # Give the service time to start before we move on
				# Look for "No space to write data|the header|end data" and restart the service
				if (str('No space to write') in line):
					os.system('echo "$(tail -n 50 ' + currentLog + ')" > ' + currentLog)
					os.system('/bin/systemctl restart dstarrepeater.service')
					time.sleep(60) # Give the service 60 secs before we re-test

	# Check the MMDVMHost Specific Services
	if os.path.isfile(mmdvmhost):
		config = ConfigParser.RawConfigParser()
		config.read(mmdvmhost)
		mmdvmhdw = config.get('Modem', 'Hardware')
		config.read('/etc/mmdvmhost')                           # modified
		dmrId = config.get('General', 'Id')                     # modified
		callsign = config.get('General', 'Callsign')            # modified

		checkprocmmdvm  = subprocess.Popen('pgrep' + ' MMDVMHost', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if mmdvmhdw and not checkprocmmdvm.stdout.readlines() and not (dmrId == '1234567'):  # modified test
			os.system('/bin/systemctl stop mmdvmhost.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start mmdvmhost.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocmmdvm.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/mmdvmhost')
		dmrAddress = config.get('DMR Network', 'Address')

		checkprocdmrgateway  = subprocess.Popen('pgrep' + ' DMRGateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (dmrAddress == '127.0.0.1') and not checkprocdmrgateway.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop dmrgateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dmrgateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdmrgateway.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/mmdvmhost')
		isEnable1 = config.get('System Fusion Network', 'Enable')
		config.read('/etc/dmr2ysf')
		isEnable2 = config.get('Enabled', 'Enabled')

		checkprocysfgateway  = subprocess.Popen('pgrep' + ' YSFGateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnable1 == 1 or isEnable2 == 1) and not checkprocysfgateway.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop ysfgateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysfgateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysfgateway.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/mmdvmhost')
		isEnable1 = config.get('P25 Network', 'Enable')
		isEnable2 = config.get('System Fusion Network', 'Enable')
		config.read('/etc/ysf2p25')
		isEnable3 = config.get('Enabled', 'Enabled')

		checkprocp25gateway  = subprocess.Popen('pgrep' + ' P25Gateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnable1 == 1 or (isEnable2 == 1 and isEnable3 == 1)) and not checkprocp25gateway.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop p25gateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start p25gateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocp25gateway.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/mmdvmhost')
		isEnable1 = config.get('System Fusion Network', 'Enable')
		config.read('/etc/dmr2ysf')
		isEnable2 = config.get('Enabled', 'Enabled')

		checkprocysfparrot  = subprocess.Popen('pgrep' + ' YSFParrot', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnable1 == 1 or isEnable2 == 1) and not checkprocysfparrot.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop ysfparrot.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysfparrot.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysfparrot.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/mmdvmhost')
		isEnable1 = config.get('P25 Network', 'Enable')
		isEnable2 = config.get('System Fusion Network', 'Enable')
		config.read('/etc/ysf2p25')
		isEnable3 = config.get('Enabled', 'Enabled')

		checkprocp25parrot  = subprocess.Popen('pgrep' + ' P25Parrot', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnable1 == 1 or (isEnable2 == 1 and isEnable3 == 1)) and not checkprocp25parrot.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop p25parrot.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start p25parrot.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocp25parrot.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/mmdvmhost')
		isEnable1 = config.get('System Fusion Network', 'Enable')
		config.read('/etc/ysf2dmr')
		isEnabled = config.get('Enabled', 'Enabled')

		checkprocysf2dmr  = subprocess.Popen('pgrep' + ' YSF2DMR', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnable1 == 1 and isEnabled == '1') and not checkprocysf2dmr.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop ysf2dmr.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysf2dmr.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysf2dmr.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/ysf2nxdn')
		isEnabled = config.get('Enabled', 'Enabled')

		checkprocysf2nxdn  = subprocess.Popen('pgrep' + ' YSF2NXDN', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnabled == '1') and not checkprocysf2nxdn.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop ysf2nxdn.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysf2nxdn.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysf2nxdn.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/ysf2p25')
		isEnabled = config.get('Enabled', 'Enabled')

		checkprocysf2p25  = subprocess.Popen('pgrep' + ' YSF2P25', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnabled == '1') and not checkprocysf2p25.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop ysf2p25.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysf2p25.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysf2p25.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/dmr2ysf')
		isEnabled = config.get('Enabled', 'Enabled')

		checkprocdmr2ysf  = subprocess.Popen('pgrep' + ' DMR2YSF', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnabled == '1') and not checkprocdmr2ysf.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop dmr2ysf.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dmr2ysf.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdmr2ysf.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/dmr2nxdn')
		isEnabled = config.get('Enabled', 'Enabled')

		checkprocdmr2nxdn  = subprocess.Popen('pgrep' + ' DMR2NXDN', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnabled == '1') and not checkprocdmr2nxdn.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop dmr2nxdn.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dmr2nxdn.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdmr2nxdn.wait()

		config = ConfigParser.RawConfigParser()
		config.read('/etc/mmdvmhost')
		isEnabled = config.get('POCSAG Network', 'Enable')

		checkprocdapnetgateway  = subprocess.Popen('pgrep' + ' DAPNETGateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if (isEnabled == '1') and not checkprocdapnetgateway.stdout.readlines():  # modified test
			os.system('/bin/systemctl stop dapnetgateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dapnetgateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdapnetgateway.wait()

	# Check the services that can be used by all systems
	config = ConfigParser.RawConfigParser()
	config.read('/etc/mmdvmhost')
	isEnabled = config.get('D-Star Network', 'Enable')

	checkprocirc  = subprocess.Popen('pgrep' + ' ircddbgatewayd', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
	if (isEnabled == '1') and not checkprocirc.stdout.readlines():  # modified test
		os.system('/bin/systemctl stop ircddbgateway.service')
		time.sleep(2) #Give the service time to stop before we move on
		os.system('/bin/systemctl start ircddbgateway.service')
		time.sleep(10) #Give the service time to start before we move on
	checkprocirc.wait()

	checkproctime = subprocess.Popen('pgrep' + ' timeserverd', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
	if (isEnabled == '1') and not os.path.isfile('/etc/timeserver.dissable') and not checkproctime.stdout.readlines():  # modified test
		os.system('/bin/systemctl stop timeserver.service')
		time.sleep(2) #Give the service time to stop before we move on
		os.system('/bin/systemctl start timeserver.service')
		time.sleep(10) #Give the service time to start before we move on
	checkproctime.wait()

	# If PiStar-Remote not enabled, dont try and keep it running
	pistarRemoteConfig = ConfigParser.RawConfigParser()
	pistarRemoteConfig.read('/etc/pistar-remote')

	isEnabled = pistarRemoteConfig.get('enable', 'enabled')
	if (isEnabled == 'true'):
		checkprocremote = subprocess.check_output(['/usr/local/sbin/pistar-remote.service', 'status'])
		if "not" in checkprocremote.decode():
			os.system('/bin/systemctl stop pistar-remote.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start pistar-remote.service')
			time.sleep(10) #Give the service time to start before we move on

#	checkprockeeper = subprocess.Popen('pgrep' + ' pistar-keeper', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
#	if not checkprockeeper.stdout.readlines():
#		os.system('service pistar-keeper start')
#		time.sleep(10) #Give the service time to start before we move on
#	checkprockeeper.wait()

	time.sleep(300) #Time to wait between service checks