genmon.py

#!/usr/bin/env python
# -------------------------------------------------------------------------------
#    FILE: genmon.py
# PURPOSE: Monitor for Generator
#
#  AUTHOR: Jason G Yates
#    DATE: 05-Oct-2016
#          23-Apr-2018
#
# MODIFICATIONS:
# -------------------------------------------------------------------------------

from __future__ import (  # For python 3.x compatibility with print function
    print_function,
)

import collections
import datetime
import getopt
import json
import os
import signal
import socket
import sys
import threading
import time

try:
    from genmonlib.custom_controller import CustomController
    from genmonlib.generac_evolution import Evolution
    from genmonlib.generac_HPanel import HPanel
    from genmonlib.generac_powerzone import PowerZone
    from genmonlib.myconfig import MyConfig
    from genmonlib.mylog import SetupLogger
    from genmonlib.mymail import MyMail
    from genmonlib.mypipe import MyPipe
    from genmonlib.myplatform import MyPlatform
    from genmonlib.mysupport import MySupport
    from genmonlib.mythread import MyThread
    from genmonlib.myweather import MyWeather
    from genmonlib.program_defaults import ProgramDefaults
except Exception as e1:
    print(
        "\n\nThis program requires the modules located in the genmonlib directory in the github repository.\n"
    )
    print(
        "Please see the project documentation at https://github.com/jgyates/genmon.\n"
    )
    print("Error: " + str(e1))
    sys.exit(2)

# ------------ Monitor class ----------------------------------------------------
class Monitor(MySupport):
    def __init__(self, ConfigFilePath=ProgramDefaults.ConfPath):
        super(Monitor, self).__init__()

        self.ProgramName = "Generator Monitor"
        self.Version = "Unknown"
        self.log = None
        self.IsStopping = False
        self.ProgramComplete = False
        if ConfigFilePath == None or ConfigFilePath == "":
            self.ConfigFilePath = ProgramDefaults.ConfPath
        else:
            self.ConfigFilePath = ConfigFilePath

        self.ConnectionList = []  # list of incoming connections for heartbeat
        # defautl values
        self.SiteName = "Home"
        self.ServerSocket = None
        self.ServerIPAddress = ""
        # server socket for nagios heartbeat and command/status
        self.ServerSocketPort = ProgramDefaults.ServerPort
        self.IncomingEmailFolder = "Generator"
        self.ProcessedEmailFolder = "Generator/Processed"

        self.FeedbackLogFile = os.path.join(self.ConfigFilePath, "feedback.json")
        self.LogLocation = ProgramDefaults.LogPath
        self.LastLogFileSize = 0
        self.NumberOfLogSizeErrors = 0
        # set defaults for optional parameters
        self.NewInstall = False  # True if newly installed or newly upgraded version
        # True if sending autoated feedback on missing information
        self.FeedbackEnabled = False  
        self.FeedbackMessages = {}

        self.MailInit = False  # set to true once mail is init
        # Flag to let the heartbeat thread know we are communicating
        self.CommunicationsActive = False

        self.Controller = None
        self.ControllerSelected = None
        self.bDisablePlatformStats = False
        self.ReadOnlyEmailCommands = False
        self.SlowCPUOptimization = False
        # weather parameters
        self.WeatherAPIKey = None
        self.WeatherLocation = None
        self.UseMetric = False
        self.WeatherMinimum = True
        self.DisableWeather = False
        self.MyWeather = None
        self.UpdateAvailable = False
        self.UpdateVersion = None

        # Time Sync Related Data
        self.bSyncTime = False  # Sync gen to system time
        self.bSyncDST = False  # sync time at DST change
        self.bDST = False  # Daylight Savings Time active if True
        # simulation
        self.Simulation = False
        self.SimulationFile = None

        self.console = SetupLogger("genmon_console", log_file="", stream=True)

        if not MySupport.PermissionsOK():
            self.LogConsole(
                "You need to have root privileges to run this script.\nPlease try again, this time using 'sudo'."
            )
            sys.exit(1)

        if not os.path.isfile(os.path.join(self.ConfigFilePath, "genmon.conf")):
            self.LogConsole(
                "Missing config file : "
                + os.path.join(self.ConfigFilePath, "genmon.conf")
            )
            sys.exit(1)
        if not os.path.isfile(os.path.join(self.ConfigFilePath, "mymail.conf")):
            self.LogConsole(
                "Missing config file : "
                + os.path.join(self.ConfigFilePath, "mymail.conf")
            )
            sys.exit(1)

        self.config = MyConfig(
            filename=os.path.join(self.ConfigFilePath, "genmon.conf"),
            section="GenMon",
            log=self.console,
        )
        # read config file
        if not self.GetConfig():
            self.LogConsole("Failure in Monitor GetConfig")
            sys.exit(1)

        # log errors in this module to a file
        self.log = SetupLogger("genmon", os.path.join(self.LogLocation, "genmon.log"))

        self.config.log = self.log

        if self.IsLoaded():  # this checks based on the port used for the API
            self.LogConsole("ERROR: genmon.py is already loaded.")
            self.LogError("ERROR: genmon.py is already loaded.")
            sys.exit(1)

        # this check is based on the file name.
        if MySupport.IsRunning(
            os.path.basename(__file__), multi_instance=self.multi_instance
        ):
            self.LogConsole("ERROR: genmon.py is already loaded.")
            self.LogError("ERROR: genmon.py is already loaded (2).")
            sys.exit(1)

        if self.NewInstall:
            self.LogError(
                "New version detected: Old = %s, New = %s"
                % (self.Version, ProgramDefaults.GENMON_VERSION)
            )
            self.Version = ProgramDefaults.GENMON_VERSION

        self.ProgramStartTime = datetime.datetime.now()  # used for com metrics
        # this will wait one day for an update, change to
        #  datetime.datetime(1, 1, 1, 0, 0) to check immediately on load
        self.LastSoftwareUpdateCheck = datetime.datetime.now()

        signal.signal(signal.SIGTERM, self.SignalClose)
        signal.signal(signal.SIGINT, self.SignalClose)

        # this allows the genmon socket interface to be intercepted and
        # triggered for actions like GPIO, etc with interfering with
        # updates via the main github repository. If genmonext.py exists, load it
        self.genmonext = None
        if os.path.isfile(
            os.path.join(os.path.dirname(os.path.realpath(__file__)), "genmonext.py")
        ):
            import genmonext

            self.genmonext = genmonext.GenmonExt(log=self.log)

        # start thread to accept incoming sockets for nagios heartbeat and command / status clients
        self.Threads["InterfaceServerThread"] = MyThread(
            self.InterfaceServerThread, Name="InterfaceServerThread"
        )

        try:
            # init mail, start processing incoming email
            self.mail = MyMail(
                monitor=True,
                incoming_folder=self.IncomingEmailFolder,
                processed_folder=self.ProcessedEmailFolder,
                incoming_callback=self.ProcessCommand,
                loglocation=self.LogLocation,
                ConfigFilePath=ConfigFilePath,
            )

            self.Threads = self.MergeDicts(self.Threads, self.mail.Threads)
            self.MailInit = True
        except Exception as e1:
            self.LogErrorLine("Error loading mail support: " + str(e1))
            sys.exit(1)

        self.FeedbackPipe = MyPipe(
            "Feedback",
            self.FeedbackReceiver,
            log=self.log,
            debug = self.debug,
            ConfigFilePath=self.ConfigFilePath,
        )
        self.Threads = self.MergeDicts(self.Threads, self.FeedbackPipe.Threads)
        self.MessagePipe = MyPipe(
            "Message",
            self.MessageReceiver,
            log=self.log,
            debug = self.debug,
            nullpipe=self.mail.DisableSNMP,
            ConfigFilePath=self.ConfigFilePath,
        )
        self.Threads = self.MergeDicts(self.Threads, self.MessagePipe.Threads)

        try:
            # Starting device connection
            if self.Simulation:
                self.LogError("Simulation Running")
            if not self.ControllerSelected == None and len(self.ControllerSelected):
                self.LogError("Selected Controller: " + str(self.ControllerSelected))
            else:
                self.ControllerSelected = "generac_evo_nexus"

            if self.ControllerSelected.lower() == "h_100":
                self.Controller = HPanel(
                    self.log,
                    newinstall=self.NewInstall,
                    simulation=self.Simulation,
                    simulationfile=self.SimulationFile,
                    message=self.MessagePipe,
                    feedback=self.FeedbackPipe,
                    config=self.config,
                )
            elif self.ControllerSelected.lower() == "powerzone":
                self.Controller = PowerZone(
                    self.log,
                    newinstall=self.NewInstall,
                    simulation=self.Simulation,
                    simulationfile=self.SimulationFile,
                    message=self.MessagePipe,
                    feedback=self.FeedbackPipe,
                    config=self.config,
                )
            elif self.ControllerSelected.lower() == "custom":
                self.Controller = CustomController(
                    self.log,
                    newinstall=self.NewInstall,
                    simulation=self.Simulation,
                    simulationfile=self.SimulationFile,
                    message=self.MessagePipe,
                    feedback=self.FeedbackPipe,
                    config=self.config,
                )
            else:
                self.Controller = Evolution(
                    self.log,
                    self.NewInstall,
                    simulation=self.Simulation,
                    simulationfile=self.SimulationFile,
                    message=self.MessagePipe,
                    feedback=self.FeedbackPipe,
                    config=self.config,
                )
            self.Threads = self.MergeDicts(self.Threads, self.Controller.Threads)

        except Exception as e1:
            self.LogErrorLine("Error opening controller device: " + str(e1))
            sys.exit(1)

        self.StartThreads()

        self.ProcessFeedbackInfo()

        # send mail to tell we are starting
        IP = self.GetNetworkIp()
        self.MessagePipe.SendMessage(
            "Generator Monitor Starting at " + self.SiteName,
            "Generator Monitor Starting at "
            + self.SiteName
            + " using IP address "
            + IP,
            msgtype="info",
        )

        self.LogError(
            "GenMon Loaded for site: "
            + self.SiteName
            + " using python "
            + str(sys.version_info.major)
            + "."
            + str(sys.version_info.minor)
            + ": VEnv: "
            + str(self.InVirtualEnvironment())
        )

    # ------------------------ Monitor::StartThreads----------------------------
    def StartThreads(self, reload=False):

        try:
            # start thread to accept incoming sockets for nagios heartbeat
            self.Threads["ComWatchDog"] = MyThread(self.ComWatchDog, Name="ComWatchDog")

            if self.bSyncDST or self.bSyncTime:  # Sync time thread
                self.Threads["TimeSyncThread"] = MyThread(
                    self.TimeSyncThread, Name="TimeSyncThread"
                )

            if (
                not self.DisableWeather
                and not self.WeatherAPIKey == None
                and len(self.WeatherAPIKey)
                and not self.WeatherLocation == None
                and len(self.WeatherLocation)
            ):
                Unit = "metric" if self.UseMetric else "imperial"
                self.MyWeather = MyWeather(
                    self.WeatherAPIKey,
                    location=self.WeatherLocation,
                    unit=Unit,
                    log=self.log,
                )
                self.Threads = self.MergeDicts(self.Threads, self.MyWeather.Threads)
        except Exception as e1:
            self.LogErrorLine("Error in StartThreads: " + str(e1))

    # -------------------- Monitor::GetConfig-----------------------------------
    def GetConfig(self):

        try:
            if self.config.HasOption("sitename"):
                self.SiteName = self.config.ReadValue("sitename")

            self.debug = self.config.ReadValue("debug", return_type=bool, default=False)

            self.multi_instance = self.config.ReadValue(
                "multi_instance", return_type=bool, default=False
            )

            if self.config.HasOption("incoming_mail_folder"):
                self.IncomingEmailFolder = self.config.ReadValue(
                    "incoming_mail_folder"
                )  # imap folder for incoming mail

            if self.config.HasOption("processed_mail_folder"):
                self.ProcessedEmailFolder = self.config.ReadValue(
                    "processed_mail_folder"
                )  # imap folder for processed mail
            #  server_port, must match value in myclient.py and check_monitor_system.py and any calling client apps
            if self.config.HasOption("server_port"):
                self.ServerSocketPort = self.config.ReadValue(
                    "server_port", return_type=int
                )

            self.ServerIPAddress = self.config.ReadValue("genmon_server_address", default = "")

            self.LogLocation = self.config.ReadValue(
                "loglocation", default=ProgramDefaults.LogPath
            )

            self.UserDefinedDataPath = self.config.ReadValue(
                "userdatalocation", default=os.path.dirname(os.path.realpath(__file__))
            )

            if self.config.HasOption("syncdst"):
                self.bSyncDST = self.config.ReadValue("syncdst", return_type=bool)
            if self.config.HasOption("synctime"):
                self.bSyncTime = self.config.ReadValue("synctime", return_type=bool)

            if self.config.HasOption("disableplatformstats"):
                self.bDisablePlatformStats = self.config.ReadValue(
                    "disableplatformstats", return_type=bool
                )

            if self.config.HasOption("simulation"):
                self.Simulation = self.config.ReadValue("simulation", return_type=bool)

            if self.config.HasOption("simulationfile"):
                self.SimulationFile = self.config.ReadValue("simulationfile")

            if self.config.HasOption("controllertype"):
                self.ControllerSelected = self.config.ReadValue("controllertype")

            if self.config.HasOption("disableweather"):
                self.DisableWeather = self.config.ReadValue(
                    "disableweather", return_type=bool
                )
            else:
                self.DisableWeather = False

            if self.config.HasOption("weatherkey"):
                self.WeatherAPIKey = self.config.ReadValue("weatherkey")

            if self.config.HasOption("weatherlocation"):
                self.WeatherLocation = self.config.ReadValue("weatherlocation")

            if self.config.HasOption("metricweather"):
                self.UseMetric = self.config.ReadValue(
                    "metricweather", return_type=bool
                )

            if self.config.HasOption("minimumweatherinfo"):
                self.WeatherMinimum = self.config.ReadValue(
                    "minimumweatherinfo", return_type=bool
                )

            if self.config.HasOption("readonlyemailcommands"):
                self.ReadOnlyEmailCommands = self.config.ReadValue(
                    "readonlyemailcommands", return_type=bool
                )

            if self.config.HasOption("optimizeforslowercpu"):
                self.SlowCPUOptimization = self.config.ReadValue(
                    "optimizeforslowercpu", return_type=bool
                )

            self.AdditionalWatchdogTime = self.config.ReadValue(
                "watchdog_addition", return_type=int, default=0
            )

            if self.config.HasOption("version"):
                self.Version = self.config.ReadValue("version")
                if not self.Version == ProgramDefaults.GENMON_VERSION:
                    self.config.WriteValue("version", ProgramDefaults.GENMON_VERSION)
                    self.NewInstall = True
            else:
                self.config.WriteValue("version", ProgramDefaults.GENMON_VERSION)
                self.NewInstall = True
                self.Version = ProgramDefaults.GENMON_VERSION
                self.config.WriteValue("install", str(datetime.datetime.now()))

            if not self.config.HasOption("install"):
                try:
                    stat = os.stat(self.config.FileName)
                    self.config.WriteValue("install", str(datetime.datetime.fromtimestamp(stat.st_atime)))
                except:
                    self.config.WriteValue("install", "Unknown")
        
            self.InstallTime = self.config.ReadValue("install", default = "Unknown")

            if self.config.HasOption("autofeedback"):
                self.FeedbackEnabled = self.config.ReadValue(
                    "autofeedback", return_type=bool
                )
            else:
                self.config.WriteValue("autofeedback", "False")
                self.FeedbackEnabled = False
            # Load saved feedback log if log is present
            if os.path.isfile(self.FeedbackLogFile):
                try:
                    with open(self.FeedbackLogFile) as infile:
                        self.FeedbackMessages = json.load(infile)
                except Exception as e1:
                    os.remove(self.FeedbackLogFile)

            self.UpdateCheck = self.config.ReadValue(
                "update_check", return_type=bool, default=True
            )
            self.UserURL = self.config.ReadValue("user_url", default="").strip()

        except Exception as e1:
            self.Console(
                "Missing config file or config file entries (genmon): " + str(e1)
            )
            return False

        return True

    # ---------------------------------------------------------------------------
    def ProcessFeedbackInfo(self):

        try:
            if self.FeedbackEnabled:
                for Key, Entry in self.FeedbackMessages.items():
                    self.MessagePipe.SendMessage(
                        "Generator Monitor Submission",
                        Entry,
                        recipient=self.MaintainerAddress,
                        files=self.GetLogFileNames(),
                        msgtype="error",
                    )
                # delete unsent Messages
                if os.path.isfile(self.FeedbackLogFile):
                    os.remove(self.FeedbackLogFile)
        except Exception as e1:
            self.LogErrorLine("Error in ProcessFeedbackInfo: " + str(e1))

    # ---------------------------------------------------------------------------
    def FeedbackReceiver(self, Message):

        try:
            FeedbackDict = {}
            FeedbackDict = json.loads(Message)
            self.SendFeedbackInfo(
                FeedbackDict["Reason"],
                Always=FeedbackDict["Always"],
                Message=FeedbackDict["Message"],
                FullLogs=FeedbackDict["FullLogs"],
                NoCheck=FeedbackDict["NoCheck"],
            )

        except Exception as e1:
            self.LogErrorLine("Error in  FeedbackReceiver: " + str(e1))
            self.LogError("Size : " + str(len(Message)))
            self.LogError("Message : " + str(Message))

    # ---------------------------------------------------------------------------
    def MessageReceiver(self, Message):

        try:
            MessageDict = {}
            MessageDict = json.loads(Message)

            self.mail.sendEmail(
                MessageDict["subjectstr"],
                MessageDict["msgstr"],
                recipient=MessageDict["recipient"],
                files=MessageDict["files"],
                deletefile=MessageDict["deletefile"],
                msgtype=MessageDict["msgtype"],
            )

        except Exception as e1:
            self.LogErrorLine("Error in  MessageReceiver: " + str(e1))

    # ---------------------------------------------------------------------------
    def SendFeedbackInfo(
        self, Reason, Always=False, Message=None, FullLogs=True, NoCheck=False
    ):
        try:
            if self.NewInstall or Always:

                CheckedSent = self.FeedbackMessages.get(Reason, "")

                if not CheckedSent == "" and not NoCheck:
                    return

                if not NoCheck:
                    self.LogError(Reason + " : " + Message)

                msgbody = "Reason = " + Reason + "\n"
                if Message != None:
                    msgbody += "Message : " + Message + "\n"
                msgbody += self.printToString(
                    self.ProcessDispatch(self.GetStartInfo(NoTile=True), "")
                )
                if not self.bDisablePlatformStats:
                    msgbody += self.printToString(
                        self.ProcessDispatch(
                            {"Platform Stats": self.GetPlatformStats()}, ""
                        )
                    )
                msgbody += self.printToString(
                    self.ProcessDispatch(
                        {"Comm Stats": self.Controller.GetCommStatus()}, ""
                    )
                )

                msgbody += "\n" + self.GetSupportData() + "\n"
                if self.FeedbackEnabled:
                    self.MessagePipe.SendMessage(
                        "Generator Monitor Submission",
                        msgbody,
                        recipient=self.MaintainerAddress,
                        files=self.GetLogFileNames(),
                        msgtype="error",
                    )

                self.FeedbackMessages[Reason] = msgbody
                # if feedback not enabled, save the log to file
                if not self.FeedbackEnabled:
                    with open(self.FeedbackLogFile, "w") as outfile:
                        json.dump(
                            self.FeedbackMessages,
                            outfile,
                            sort_keys=True,
                            indent=4,
                            ensure_ascii=False,
                        )
        except Exception as e1:
            self.LogErrorLine("Error in SendFeedbackInfo: " + str(e1))

    # ---------- Monitor::EmailSendIsEnabled-------------------------------------
    def EmailSendIsEnabled(self):

        EmailThread = self.Threads.get("SendMailThread", None)
        if EmailThread == None:
            return False
        return True

    # ---------- Monitor::GetSupportData-----------------------------------------
    def GetSupportData(self):

        SupportData = collections.OrderedDict()
        try:
            SupportData["Program Run Time"] = self.GetProgramRunTime()
            SupportData["Install"] = self.InstallTime
            SupportData["Monitor Health"] = self.GetSystemHealth()
            SupportData["Controller Selected"] = self.ControllerSelected.lower()
            SupportData["StartInfo"] = self.GetStartInfo(NoTile=True)
            SupportData["Comm Stats"] = self.Controller.GetCommStatus()
            if not self.bDisablePlatformStats:
                SupportData["PlatformStats"] = self.GetPlatformStats()
            #SupportData["Data"] = self.Controller.DisplayRegisters(AllRegs=True, DictOut=True)
            # Raw Modbus data
            SupportData["Holding"] = self.Controller.Holding
            SupportData["Strings"] = self.Controller.Strings
            SupportData["FileData"] = self.Controller.FileData
            SupportData["Coils"] = self.Controller.Coils
            SupportData["Inputs"] = self.Controller.Inputs
        except Exception as e1:
            self.LogErrorLine("Error in GetSupportData: " + str(e1))

        try:
            # indent 4 will keep some mail servers from having problems.
            return json.dumps(SupportData, indent=4, sort_keys=False)
        except Exception as e1:
            self.LogErrorLine("Error in GetSupportData (2): " + str(e1))
            return "Error Getting JSON data: " + str(e1)

    # ---------- Monitor::GetLogFileNames----------------------------------------
    def GetLogFileNames(self):

        try:
            LogList = []
            FilesToSend = [
                "genmon.log",
                "genserv.log",
                "mymail.log",
                "myserial.log",
                "mymodbus.log",
                "gengpio.log",
                "gengpioin.log",
                "gensms.log",
                "gensms_modem.log",
                "genmqtt.log",
                "genmqttin.log",
                "genpushover.log",
                "gensyslog.log",
                "genloader.log",
                "myserialtcp.log",
                "genlog.log",
                "genslack.log",
                "gencallmebot.log",
                "genexercise.log",
                "genemail2sms.log",
                "gencentriconnect.log",
                "gentankutil.log",
                "genalexa.log",
                "gensnmp.log",
                "gentemp.log",
                "gentankdiy.log",
                "gengpioledblink.log",
                "gencthat.log",
                "genmopeka.log",
                "gencustomgpio.log",
                "gensms_voip.log",
            ]
            DataFilesToSend = [
                "update.txt"        # time stamp of software update
            ]
            # Files in /var/log
            for File in FilesToSend:
                LogFile = self.LogLocation + File
                if os.path.isfile(LogFile):
                    LogList.append(LogFile)
            # Non settings realated files in conf folder (typically /etc/genmon)
            for File in DataFilesToSend:
                LogFile = self.ConfigFilePath + File
                if os.path.isfile(LogFile):
                    LogList.append(LogFile)
            return LogList
        except Exception as e1:
            return None

    # ---------- Monitor::SendSupportInfo----------------------------------------
    def SendSupportInfo(self, SendLogs=True):

        try:
            if not self.EmailSendIsEnabled():
                self.LogError("Error in SendSupportInfo: send email is not enabled")
                return "Send Email is not enabled."

            msgbody = ""
            msgbody += self.printToString(
                self.ProcessDispatch(self.GetStartInfo(NoTile=True), "")
            )
            if not self.bDisablePlatformStats:
                msgbody += self.printToString(
                    self.ProcessDispatch(
                        {"Platform Stats": self.GetPlatformStats()}, ""
                    )
                )
            msgbody += self.printToString(
                self.ProcessDispatch(
                    {"Comm Stats": self.Controller.GetCommStatus()}, ""
                )
            )

            #msgbody += self.Controller.DisplayRegisters(AllRegs=True)

            # get data in JSON format
            msgbody += "\n" + self.GetSupportData() + "\n"
            msgtitle = "Generator Monitor Log File Submission"
            if SendLogs == True:
                LogList = self.GetLogFileNames()
            else:
                msgtitle = "Generator Monitor Register Submission"
                LogList = None
            self.MessagePipe.SendMessage(
                msgtitle,
                msgbody,
                recipient=self.MaintainerAddress,
                files=LogList,
                msgtype="error",
            )
            return "Log files submitted"
        except Exception as e1:
            self.LogErrorLine("Error in SendSupportInfo: " + str(e1))

    # ---------- Send message ---------------------------------------------------
    def SendMessage(self, CmdString):
        try:
            self.LogDebug("ENTER SendMessage")
            if CmdString == None or CmdString == "":
                return "Error: invalid command in SendMessage"

            CmdList = CmdString.split("=")
            if len(CmdList) != 2:
                self.LogError(
                    "Validation Error: Error parsing command string in SendMessage (parse): "
                    + CmdString
                )
                return "Error in SendMessage"

            data = json.loads(CmdList[1])
            msgtitle = self.SiteName + ": " + data["title"]
            if not "onlyonce" in data:
                onlyonce = False
            else:
                onlyonce = data["onlyonce"]
            if not "oncedaily" in data:
                oncedaily = False
            else:
                oncedaily = data["oncedaily"]
            self.LogDebug("SENDMSG:" + str(data))
            self.MessagePipe.SendMessage(
                msgtitle, data["body"], msgtype=data["type"], onlyonce=onlyonce, oncedaily=oncedaily
            )
            return "OK"
        except Exception as e1:
            self.LogErrorLine("Error in SendMessage: " + str(e1))
        return "OK"

    # ---------- process command from email and socket --------------------------
    def ProcessCommand(self, command, fromsocket=False):

        LocalError = False
        if isinstance(command, bytes):
            command = command.decode("utf-8")
        try:
            msgsubject = "Genmon Command Response at " + self.SiteName
            if not fromsocket:
                msgbody = "\n"
            else:
                msgbody = ""

            if (len(command)) == 0:
                msgsubject = "Error in Genmon Command (Lenght is zero)"
                msgbody += "Invalid GENERATOR command: zero length command."
                LocalError = True

            if not LocalError:
                if not command.lower().startswith("generator:"):
                    msgsubject = "Error in Genmon Command (command prefix)"
                    msgbody += 'Invalid GENERATOR command: all commands must be prefixed by "generator: "'
                    LocalError = True

            if LocalError:
                if not fromsocket:
                    self.MessagePipe.SendMessage(msgsubject, msgbody, msgtype="error")
                    return ""  # ignored by email module
                else:
                    msgbody += "EndOfMessage"
                    return msgbody

            if self.genmonext != None:
                try:
                    self.genmonext.PreProcessCommand(command)
                except Exception as e1:
                    self.LogErrorLine("Error Calling GenmonExt:PreProcessCommand: " + str(e1))

            if command.lower().startswith("generator:"):
                command = command[len("generator:") :]

            CommandDict = {
                "registers": [self.Controller.DisplayRegisters,(False,),False,],  # display registers
                "allregs": [self.Controller.DisplayRegisters,(True,),False,],  # display registers
                "logs": [self.Controller.DisplayLogs, (True, False), False],
                "status": [self.Controller.DisplayStatus,(),False,],  # display decoded generator info
                "maint": [self.Controller.DisplayMaintenance, (), False],
                "monitor": [self.DisplayMonitor, (), False],
                "outage": [self.Controller.DisplayOutage, (), False],
                "settime": [self.StartTimeThread, (), False],  # set time and date
                "setexercise": [self.Controller.SetGeneratorExerciseTime,(command.lower(),),False,],
                "setquiet": [self.Controller.SetGeneratorQuietMode,(command.lower(),),False,],
                "setremote": [
                    self.Controller.SetGeneratorRemoteCommand,
                    (command.lower(),),
                    False,
                ],
                "testcommand": [self.Controller.TestCommand, (command.lower(),), False],
                "network_status": [self.InternetConnected, (), False],
                "help": [self.DisplayHelp, (), False],  # display help screen
                ## These commands are used by the web / socket interface only
                "power_log_json": [
                    self.Controller.GetPowerHistory,
                    (command.lower(),False, True),
                    True,
                ],
                "power_log_clear": [self.Controller.ClearPowerLog, (), True],
                "fuel_log_clear": [self.Controller.ClearFuelLog, (), True],
                "start_info_json": [self.GetStartInfo, (), True],
                "registers_json": [
                    self.Controller.DisplayRegisters,
                    (False, True),
                    True,
                ],  # display registers
                "allregs_json": [
                    self.Controller.DisplayRegisters,
                    (True, True),
                    True,
                ],  # display registers
                "logs_json": [self.Controller.DisplayLogs, (True, True), True],
                "status_json": [self.Controller.DisplayStatus, (True,), True],
                "status_num_json": [self.Controller.DisplayStatus, (True, True), True],
                "maint_json": [self.Controller.DisplayMaintenance, (True,), True],
                "maint_num_json": [self.Controller.DisplayMaintenance, (True, True), True],
                "monitor_json": [self.DisplayMonitor, (True,), True],
                "monitor_num_json": [self.DisplayMonitor, (True, True), True],
                "weather_json": [self.DisplayWeather, (True,), True],
                "outage_json": [self.Controller.DisplayOutage, (True,), True],
                "outage_num_json": [self.Controller.DisplayOutage, (True, True), True],
                "gui_status_json": [self.GetStatusForGUI, (), True],
                "get_maint_log_json": [self.Controller.GetMaintLogJSON, (), True],
                "add_maint_log": [
                    self.Controller.AddEntryToMaintLog,
                    (command,),
                    True,
                ],  # Do not do command.lower() since this input is JSON
                "delete_row_maint_log": [
                    self.Controller.DeleteMaintLogRow,
                    (command.lower(),),
                    True,
                ],
                "edit_row_maint_log": [
                    self.Controller.EditMaintLogRow,
                    (command,),
                    True,
                ],  # Do not do command.lower() since this input is JSON
                "clear_maint_log": [self.Controller.ClearMaintLog, (), True],
                "getsitename": [self.GetSiteName, (), True],
                "getbase": [
                    self.Controller.GetBaseStatus,
                    (),
                    True,
                ],  #  (UI changes color based on exercise, running , ready status)
                "gethealth": [self.GetSystemHealth, (), True],
                "getregvalue": [
                    self.Controller.GetRegValue,
                    (command.lower(),),
                    True,
                ],  # only used for debug purposes, read a cached register value
                "readregvalue": [
                    self.Controller.ReadRegValue,
                    (command.lower(),),
                    True,
                ],  # only used for debug purposes, Read Register Non Cached
                # only used for debug purposes, Read Register Non Cached
                "writeregvalue": [self.Controller.WriteRegValue,(command.lower(),),True,],  
                # only used for debug purposes. If a thread crashes it tells you the thread name
                "getdebug": [self.GetDeadThreadName,(),True,],  
                "sendregisters": [self.SendSupportInfo, (False,), True],
                "sendlogfiles": [self.SendSupportInfo, (True,), True],
                "support_data_json": [self.GetSupportData, (), True],
                "set_tank_data": [self.Controller.SetExternalTankData, (command,), True],
                "set_sensor_data": [self.Controller.SetExternalSensorData,(command,),True,],
                "set_external_gauge_data": [self.Controller.SetExternalGaugeData,(command,),True,],
                "set_power_data": [self.Controller.SetExternalCTData, (command,), True],
                "notify_message": [self.SendMessage, (command,), True],
                "getreglabels_json": [self.Controller.GetRegisterLabels, (), True],
                "set_button_command": [self.Controller.SetCommandButton, (command,), True]
            }

            CommandList = command.split(" ")
        except Exception as e1:
            msgbody = "Error in ProcessCommand: " + str(e1)
            msgbody += "EndOfMessage"
            return msgbody

        ValidCommand = False
        try:
            for item in CommandList:
                if not len(item):
                    continue
                item = item.strip()
                LookUp = item
                if "=" in item:
                    BaseCmd = item.split("=")
                    LookUp = BaseCmd[0]
                # check if we disallow write commands via email
                if (
                    self.ReadOnlyEmailCommands
                    and not fromsocket
                    and LookUp in ["settime", "setexercise", "setquiet", "setremote"]
                ):
                    continue

                ExecList = CommandDict.get(LookUp.lower(), None)
                if ExecList == None:
                    continue
                if ExecList[0] == None:
                    continue
                if not fromsocket and ExecList[2]:
                    continue
                # Execute Command
                ReturnMessage = ExecList[0](*ExecList[1])

                ValidCommand = True

                if LookUp.lower().endswith("_json") and not isinstance(
                    ReturnMessage, str
                ):
                    msgbody += json.dumps(ReturnMessage, sort_keys=False)
                else:
                    msgbody += ReturnMessage

                if not fromsocket:
                    msgbody += "\n"
        except Exception as e1:
            self.LogErrorLine("Error Processing Commands: " + command + ": " + str(e1))

        if not ValidCommand:
            msgbody += "No valid command recognized."
        if not fromsocket:
            self.MessagePipe.SendMessage(msgsubject, msgbody, msgtype="warn")
            return ""  # ignored by email module
        else:
            msgbody += "EndOfMessage"
            return msgbody

    # ------------ Monitor::DisplayHelp -----------------------------------------
    def DisplayHelp(self):

        outstring = ""
        outstring += "Help:\n"
        outstring += self.printToString("\nCommands:")
        outstring += self.printToString(
            "   status      - display engine and line information"
        )
        outstring += self.printToString(
            "   maint       - display maintenance and service information"
        )
        outstring += self.printToString(
            "   outage      - display current and last outage (since program launched)"
        )
        outstring += self.printToString(
            "                       info, also shows utility min and max values"
        )
        outstring += self.printToString(
            "   monitor     - display communication statistics and monitor health"
        )
        outstring += self.printToString(
            "   logs        - display all alarm, on/off, and maintenance logs"
        )
        outstring += self.printToString(
            "   registers   - display contents of registers being monitored"
        )
        outstring += self.printToString(
            "   settime     - set generator time to system time"
        )
        outstring += self.printToString(
            "   setexercise - set the exercise time of the generator. "
        )
        outstring += self.printToString(
            "                      i.e. setexercise=Monday,13:30,Weekly"
        )
        outstring += self.printToString(
            "                  if Enhanced Exercise Frequency is supported by"
        )
        outstring += self.printToString("                  your generator:")
        outstring += self.printToString(
            "                      i.e. setexercise=Monday,13:30,BiWeekly"
        )
        outstring += self.printToString(
            "                      i.e. setexercise=15,13:30,Monthly"
        )
        outstring += self.printToString(
            "   setquiet    - enable or disable exercise quiet mode, "
        )
        outstring += self.printToString(
            "                      i.e.  setquiet=on or setquiet=off"
        )
        outstring += self.printToString(
            "   setremote   - issue remote command. format is setremote=command, "
        )
        outstring += self.printToString(
            "                      where command is start, stop, starttransfer,"
        )
        outstring += self.printToString(
            "                      startexercise. i.e. setremote=start"
        )
        outstring += self.printToString("   help        - Display help on commands")
        outstring += self.printToString("   exit        - Exit this program")
        outstring += self.printToString(
            "   sendlogfiles - Send log files to the developer if outbound email is setup."
        )
        outstring += self.printToString("                      is setup.")

        outstring += self.printToString("\n")

        outstring += self.printToString(
            "To clear the Alarm/Warning message, press OFF on the control panel keypad"
        )
        outstring += self.printToString(
            "followed by the ENTER key. To access Dealer Menu on the Evolution"
        )
        outstring += self.printToString(
            "controller, from the top menu selection (SYSTEM, DATE/TIME,BATTERY, SUB-MENUS)"
        )
        outstring += self.printToString(
            "enter UP UP ESC DOWN UP ESC UP, then go to the dealer menu and press enter."
        )
        outstring += self.printToString(
            "For liquid cooled models a level 2 dealer code can be entered, ESC UP UP DOWN"
        )
        outstring += self.printToString(
            "DOWN ESC ESC, then navigate to the dealer menu and press enter."
        )
        outstring += self.printToString(
            "Passcode for Nexus controller is ESC, UP, UP ESC, DOWN, UP, ESC, UP, UP, ENTER."
        )
        outstring += self.printToString("\n")

        return outstring

    # ------------ Monitor::GetProgramRunTime -----------------------------------
    def GetProgramRunTime(self):
        try:
            ProgramRunTime = datetime.datetime.now() - self.ProgramStartTime
            outstr = str(ProgramRunTime).split(".")[
                0
            ]  # remove microseconds from string
            return self.ProgramName + " running for " + outstr + "."
        except Exception as e1:
            self.LogErrorLine("Error in GetProgramRunTime:" + str(e1))
            return "Unknown"

    # ------------ Monitor::GetWeatherData --------------------------------------
    def GetWeatherData(self, ForUI=False, JSONNum=False):

        if self.MyWeather == None:
            return None

        ReturnData = self.MyWeather.GetWeather(minimum=self.WeatherMinimum, ForUI=ForUI, JSONNum=JSONNum)

        if not len(ReturnData):
            return None
        return ReturnData

    # ------------ Monitor::GetUserDefinedData ----------------------------------
    # this assumes one json object, the file can be formatted (i.e. on multiple
    # lines) or can be on a single line
    def GetUserDefinedData(self, JSONNum=False):

        try:
            FileName = os.path.join(self.UserDefinedDataPath, "userdefined.json")

            if not os.path.isfile(FileName):
                return None

            if os.path.getsize(FileName) == 0:
                return None

            with open(FileName) as f:
                data = json.load(f, object_pairs_hook=collections.OrderedDict)
            return data
        except Exception as e1:
            self.LogErrorLine("Error in GetUserDefinedData: " + str(e1))
        return None

    # ------------ Monitor::DisplayWeather --------------------------------------
    def DisplayWeather(self, DictOut=False):

        WeatherData = collections.OrderedDict()

        try:
            ReturnData = self.GetWeatherData()
            if not ReturnData == None and len(ReturnData):
                WeatherData["Weather"] = ReturnData

            if not DictOut:
                return self.printToString(self.ProcessDispatch(WeatherData, ""))
        except Exception as e1:
            self.LogErrorLine("Error in DisplayWeather: " + str(e1))

        return WeatherData

    # ------------ Monitor::DisplayMonitor --------------------------------------
    def DisplayMonitor(self, DictOut=False, JSONNum=False):

        try:
            Monitor = collections.OrderedDict()
            MonitorData = []
            Monitor["Monitor"] = MonitorData
            GenMonStats = []
            SerialStats = []
            MonitorData.append({"Generator Monitor Stats": GenMonStats})
            MonitorData.append({"Communication Stats": self.Controller.GetCommStatus()})

            GenMonStats.append({"Monitor Health": self.GetSystemHealth()})
            GenMonStats.append(
                {"Controller": self.Controller.GetController(Actual=False)}
            )

            GenMonStats.append({"Run time": self.GetProgramRunTime()})
            if self.Controller.PowerMeterIsSupported():
                GenMonStats.append(
                    {"Power log file size": self.Controller.GetPowerLogFileDetails()}
                )
            GenMonStats.append(
                {"Generator Monitor Version": ProgramDefaults.GENMON_VERSION}
            )
            GenMonStats.append(
                {"Update Available": "Yes" if self.UpdateAvailable else "No"}
            )

            if not self.bDisablePlatformStats:
                PlatformStats = self.GetPlatformStats(JSONNum=False)
                if not PlatformStats == None:
                    MonitorData.append({"Platform Stats": PlatformStats})

            WeatherData = self.GetWeatherData()
            if not WeatherData == None and len(WeatherData):
                MonitorData.append({"Weather": WeatherData})

            UserData = self.GetUserDefinedData()
            if UserData != None and len(UserData):
                try:
                    MonitorData.append({"External Data": UserData})
                except Exception as e1:
                    self.LogErrorLine("Error in appending user data: " + str(e1))
            if not DictOut:
                return self.printToString(self.ProcessDispatch(Monitor, ""))
        except Exception as e1:
            self.LogErrorLine("Error in DisplayMonitor: " + str(e1))
        return Monitor

    # ------------ Monitor::GetStartInfo-----------------------------------------
    def GetStartInfo(self, NoTile=False):

        StartInfo = collections.OrderedDict()
        StartInfo["version"] = ProgramDefaults.GENMON_VERSION
        StartInfo["sitename"] = self.SiteName
        StartInfo["install"] = self.InstallTime
        StartInfo["python"] = (
            str(sys.version_info.major) + "." + str(sys.version_info.minor)
        )
        StartInfo["platform"] = str(sys.platform)
        try:
            import platform

            StartInfo["python"] = str(platform.python_version())

            Platform = MyPlatform(log=self.log, usemetric=True, debug = self.debug)
            StartInfo["os_bits"] = Platform.PlatformBitDepth()

        except Exception as e1:
            self.LogErrorLine("Error in GetStartInfo: " + str(e1))
            pass
        try:
            import time

            if self.is_dst:
                StartInfo["zone"] = time.tzname[1]
            else:
                StartInfo["zone"] = time.tzname[0]
        except:
            pass
        ControllerStartInfo = self.Controller.GetStartInfo(NoTile=NoTile)
        StartInfo = self.MergeDicts(StartInfo, ControllerStartInfo)
        return StartInfo

    # ------------ Monitor::GetStatusForGUI -------------------------------------
    def GetStatusForGUI(self):

        Status = {}

        Status["SystemHealth"] = self.GetSystemHealth()
        Status["UnsentFeedback"] = str(os.path.isfile(self.FeedbackLogFile))

        if not self.bDisablePlatformStats:
            PlatformStats = self.GetPlatformStats(usemetric=True)
            if not PlatformStats == None and len(PlatformStats):
                Status["PlatformStats"] = PlatformStats
        WeatherData = self.GetWeatherData(ForUI=True)
        if not WeatherData == None and len(WeatherData):
            Status["Weather"] = WeatherData
        # Monitor Time
        Status["MonitorTime"] = datetime.datetime.now().strftime("%m/%d/%Y %H:%M")
        # Engine run hours
        Status["RunHours"] = self.Controller.GetRunHours()
        Status["AltDateformat"] = self.Controller.bAlternateDateFormat
        Status["version"] = ProgramDefaults.GENMON_VERSION
        ReturnDict = self.MergeDicts(Status, self.Controller.GetStatusForGUI())

        return ReturnDict

    # -------------Monitor::GetSystemHealth--------------------------------------
    #   returns the health of the monitor program
    def GetSystemHealth(self):

        outstr = ""
        if not self.Controller.InitComplete:
            outstr += "System Initializing. "
        if not self.AreThreadsAlive():
            outstr += " Threads are dead. "
        if not self.CommunicationsActive:
            outstr += " Not receiving data. "
        if not self.LogFileIsOK():
            outstr += " Log file is reporting errors."

        if len(outstr) == 0:
            outstr = "OK"
        return outstr

    # ----------  Monitor::StartTimeThread---------------------------------------
    def StartTimeThread(self):

        # This is done is a separate thread as not to block any return email processing
        # since we attempt to sync with generator time
        MyThread(self.Controller.SetGeneratorTimeDate, Name="SetTimeThread")
        return "Time Set: Command Sent\n"

    # ----------  Monitor::TimeSyncThread----------------------------------------
    def TimeSyncThread(self):

        self.bDST = self.is_dst()  # set initial DST state

        time.sleep(0.25)
        while True:
            if self.WaitForExit("TimeSyncThread", 1):  # ten min
                return
            if self.Controller.InitComplete:
                break

        # if we are not always syncing, then set the time once
        if not self.bSyncTime:
            self.StartTimeThread()

        while True:

            if self.bSyncDST:
                if self.bDST != self.is_dst():  # has DST changed?
                    self.bDST = self.is_dst()  # update Flag
                    # time changed so some comm stats may be off
                    self.Controller.ResetCommStats()
                    # set new time
                    self.StartTimeThread()  # start settime thread
                    self.MessagePipe.SendMessage(
                        "Generator Time Update at " + self.SiteName,
                        "Time updated due to daylight savings time change",
                        msgtype="info",
                    )
                    self.LogError("DST change")

            if self.bSyncTime:
                # update gen time
                self.StartTimeThread()

            if self.WaitForExit("TimeSyncThread", 60 * 60):  # 1 hour
                return

    # ----------  Monitor::is_dst------------------------------------------------
    def is_dst(self):
        # Determine whether or not Daylight Savings Time (DST) is currently in effect
        t = time.localtime()
        isdst = t.tm_isdst
        return isdst != 0

    # ----------  Monitor::ComWatchDog-------------------------------------------
    # ----------  monitors receive data status to make sure we are still communicating
    def ComWatchDog(self):

        self.CommunicationsActive = False
        time.sleep(0.25)

        NoticeSent = False
        LastActiveTime = datetime.datetime.now()
        counter = 0
        while True:
            if self.WaitForExit("ComWatchDog", 1):
                return
            if counter > 30:
                self.LogError(
                    "WARNING: Initialization not complete after 30 seconds, possible communication failure. Check cabling."
                )
                break
            counter += 1
            if self.Controller.InitComplete:
                break

        if self.Controller.ModBus.UseTCP:
            WatchDogPollTime = 8.0
        else:
            WatchDogPollTime = 2.0

        try:
            WatchDogPollTime += float(
                self.Controller.ModBus.ModBusPacketTimoutMS / 1000
            )
        except Exception as e1:
            self.LogErrorLine("Error in ComWatchDog: " + str(e1))

        WatchDogPollTime += self.AdditionalWatchdogTime

        while True:
            try:
                # check for software update
                self.CheckSoftwareUpdate()
                self.CommunicationsActive = self.Controller.ComminicationsIsActive()

                if self.CommunicationsActive:
                    LastActiveTime = datetime.datetime.now()
                    if NoticeSent:
                        NoticeSent = False
                        msgbody = (
                            "Generator Monitor communications with the controller has been restored at "
                            + self.SiteName
                        )
                        msgbody += "\n" + self.DisplayMonitor()
                        self.MessagePipe.SendMessage(
                            "Generator Monitor Communication Restored at "
                            + self.SiteName,
                            msgbody,
                            msgtype="error",
                        )
                else:
                    if self.GetDeltaTimeMinutes(
                        datetime.datetime.now() - LastActiveTime
                    ) > (1 + self.AdditionalWatchdogTime):
                        if not NoticeSent:
                            NoticeSent = True
                            msgbody = (
                                "Generator Monitor is not communicating with the controller at "
                                + self.SiteName
                            )
                            msgbody += "\n" + self.DisplayMonitor()
                            self.MessagePipe.SendMessage(
                                "Generator Monitor Communication WARNING at "
                                + self.SiteName,
                                msgbody,
                                msgtype="error",
                            )

            except Exception as e1:
                self.LogErrorLine("Error in ComWatchDog: " + str(e1))

            if self.WaitForExit("ComWatchDog", WatchDogPollTime):
                return

    # ---------- Monitor::CheckSoftwareUpdate------------------------------------
    def CheckSoftwareUpdate(self):

        if not self.UpdateCheck:
            return
        try:
            if (
                self.GetDeltaTimeMinutes(
                    datetime.datetime.now() - self.LastSoftwareUpdateCheck
                )
                > 1440
            ):  # check every day
                self.LastSoftwareUpdateCheck = datetime.datetime.now()
                # Do the check
                try:
                    url = "https://raw.githubusercontent.com/jgyates/genmon/master/genmonlib/program_defaults.py"
                    if sys.version_info[0] < 3:
                        # Fall back to Python 2's urllib2
                        from urllib2 import urlopen
                    else:
                        # For Python 3.0 and later
                        from urllib.request import urlopen

                    data = urlopen(url).read(4000)  # read only first 4000 chars
                    data = data.decode("ascii")
                    data = data.split("\n")  # then split it into lines

                    for line in data:

                        if 'GENMON_VERSION = "V' in line:
                            import re

                            quoted = re.compile('"([^"]*)"')
                            for value in quoted.findall(line):
                                if value != ProgramDefaults.GENMON_VERSION:
                                    # Update Available
                                    title = (
                                        self.ProgramName
                                        + " Software Update "
                                        + value
                                        + " is available for site "
                                        + self.SiteName
                                    )
                                    msgbody = (
                                        "\nA software update is available for the "
                                        + self.ProgramName
                                        + ". The new version ("
                                        + value
                                        + ") can be updated on the About page of the web interface. The current version installed is "
                                        + ProgramDefaults.GENMON_VERSION
                                        + ". You can disable this email from being sent on the Settings page."
                                    )
                                    if len(self.UserURL):
                                        msgbody += (
                                            "\n\nWeb Interface URL: " + self.UserURL
                                        )
                                    msgbody += "\n\nChange Log: https://raw.githubusercontent.com/jgyates/genmon/master/changelog.md"
                                    self.MessagePipe.SendMessage(
                                        title, msgbody, msgtype="info", onlyonce=True
                                    )
                                    self.UpdateAvailable = True
                                    self.UpdateVersion = value

                except Exception as e1:
                    self.LogErrorLine("Error checking for software update: " + str(e1))

        except Exception as e1:
            self.LogErrorLine("Error in CheckSoftwareUpdate: " + str(e1))

    # ---------- Monitor::LogFileIsOK--------------------------------------------
    def LogFileIsOK(self):

        try:
            if not self.Controller.InitComplete:
                return True

            LogFile = os.path.join(self.LogLocation, "genmon.log")

            LogFileSize = os.path.getsize(LogFile)
            if LogFileSize <= self.LastLogFileSize:  # log is unchanged or has rotated
                self.LastLogFileSize = LogFileSize
                self.NumberOfLogSizeErrors = 0
                return True

            LogFileSizeDiff = LogFileSize - self.LastLogFileSize
            self.LastLogFileSize = LogFileSize
            if LogFileSizeDiff >= 100:
                self.NumberOfLogSizeErrors += 1
                if self.NumberOfLogSizeErrors > 3:
                    return False
            else:
                self.NumberOfLogSizeErrors = 0
            return True
        except Exception as e1:
            self.LogErrorLine("Error in LogFileIsOK: " + str(e1))
        return True

    # ----------  Monitor::SocketWorkThread-------------------------------------
    #  This thread spawns for each connection established by a client
    #  in InterfaceServerThread
    def SocketWorkThread(self, conn):

        try:

            statusstr = ""
            if self.Controller == None:
                outstr = "WARNING: System Initializing"
                conn.sendall(outstr.encode())
            else:
                if self.Controller.SystemInAlarm():
                    statusstr += "CRITICAL: System in alarm! "
                HealthStr = self.GetSystemHealth()
                if HealthStr != "OK":
                    statusstr += "WARNING: " + HealthStr
                if statusstr == "":
                    statusstr = "OK "

                outstr = statusstr + ": " + self.Controller.GetOneLineStatus()
                conn.sendall(outstr.encode())

            while True:
                try:
                    data = conn.recv(2098152)  # max json string size plus 1000
                    if len(data):
                        if self.Controller == None:
                            outstr = "Retry, System Initializing"
                        else:
                            outstr = self.ProcessCommand(data, True)
                        conn.sendall(outstr.encode("utf-8"))
                    else:
                        # socket closed remotely
                        break
                except socket.timeout:
                    if self.IsStopping:
                        break
                    continue
                except socket.error as msg:
                    try:
                        self.ConnectionList.remove(conn)
                        conn.close()
                    except:
                        pass
                    break

        except socket.error as msg:
            self.LogError("Error in SocketWorkThread: " + str(msg))
            pass

        try:
            self.ConnectionList.remove(conn)
            conn.close()
        except:
            pass
        # end SocketWorkThread

    # ----------  interface for heartbeat server thread -------------------------
    def InterfaceServerThread(self):

        # create an INET, STREAMing socket
        self.ServerSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # set some socket options so we can resuse the port
        self.ServerSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.ServerSocket.settimeout(0.5)
        # bind the socket to a host, and a port
        self.ServerSocket.bind((self.ServerIPAddress, self.ServerSocketPort))
        # become a server socket
        self.ServerSocket.listen(5)

        # wait to accept a connection - blocking call
        while True:
            try:
                conn, addr = self.ServerSocket.accept()
                # self.LogError('Connected with ' + addr[0] + ':' + str(addr[1]))
                conn.settimeout(0.5)
                self.ConnectionList.append(conn)
                SocketThread = threading.Thread(
                    target=self.SocketWorkThread, args=(conn,), name="SocketWorkThread"
                )
                SocketThread.daemon = True
                SocketThread.start()  # start server thread
            except socket.timeout:
                if self.IsStopping:
                    break
                continue
            except Exception as e1:
                if self.IsStopping:
                    break
                self.LogErrorLine("Exception in InterfaceServerThread" + str(e1))
                if self.WaitForExit("InterfaceServerThread", 0.5):
                    break
                continue

        if self.ServerSocket != None:
            if len(self.ConnectionList):
                try:
                    self.ServerSocket.shutdown(socket.SHUT_RDWR)
                except:
                    pass
            self.ServerSocket.close()
            self.ServerSocket = None
        #

    # ----------Monitor::SignalClose--------------------------------------------
    def SignalClose(self, signum, frame):

        self.Close()
        sys.exit(1)

    # ---------------------Monitor::Close----------------------------------------
    def Close(self):

        # we dont really care about the errors that may be generated on shutdown
        try:
            self.IsStopping = True

            try:
                if self.MyWeather != None:
                    self.MyWeather.Close()
            except:
                pass

            try:
                if self.bSyncDST or self.bSyncTime:
                    self.KillThread("TimeSyncThread")
            except:
                pass

            try:
                self.KillThread("ComWatchDog")
            except:
                pass

            try:
                if not self.Controller == None:
                    self.Controller.Close()
            except:
                pass

            #
            try:
                self.mail.Close()
            except:
                pass
            try:
                for item in self.ConnectionList:
                    try:
                        item.close()
                    except:
                        continue
                    self.ConnectionList.remove(item)
            except:
                pass

            try:
                if self.ServerSocket != None:
                    self.ServerSocket.shutdown(socket.SHUT_RDWR)
                    self.ServerSocket.close()
                self.KillThread("InterfaceServerThread")
            except:
                pass

            try:
                self.FeedbackPipe.Close()
            except:
                pass
            try:
                self.MessagePipe.Close()
            except:
                pass

            # Tell any remaining threads to stop
            for name, object in self.Threads.items():
                try:
                    if self.Threads[name].IsAlive():
                        self.Threads[name].Stop()
                except Exception as e1:
                    self.LogErrorLine(
                        "Error killing thread in Monitor Close: " + name + ":" + str(e1)
                    )

        except Exception as e1:
            self.LogErrorLine("Error Closing Monitor: " + str(e1))

        with self.CriticalLock:
            self.LogError("Generator Monitor Shutdown")

        try:

            self.ProgramComplete = True
            sys.exit(0)
        except:
            pass


# ------------------- Command-line interface for monitor ------------------------
if __name__ == "__main__":  #

    try:
        ConfigFilePath = ProgramDefaults.ConfPath
        opts, args = getopt.getopt(sys.argv[1:], "c:", ["configpath="])
    except getopt.GetoptError:
        print("Invalid command line argument.")
        sys.exit(2)

    for opt, arg in opts:

        if opt in ("-c", "--configpath"):
            ConfigFilePath = arg
            ConfigFilePath = ConfigFilePath.strip()

    # Start things up
    MyMonitor = Monitor(ConfigFilePath=ConfigFilePath)

    try:
        while not MyMonitor.ProgramComplete:
            time.sleep(0.01)
        sys.exit(0)
    except:
        sys.exit(1)