Add RAM_TOTAL, Change monitor.sql

Author: QuangTung97

lb.py

@@ -22,59 +22,65 @@ def leastLoadServer(server_repo, ssample_repo, pred_repo):
         # Total cpu
         server_dict[server] = server_cpu + sum_avg_cpu
 
-    # Print the server with the lowest load
-    print(min(server_dict, key=server_dict.get))
+    return min(server_dict, key=server_dict.get)
 
 
 def loadBalance(uid, server_repo, ssample_repo, pred_repo):
-    if pred_repo.exist(uid):
-        # Prediction Record of UID
-        pred = pred_repo.get(uid)
-        last_hour = pred.last_login - datetime.timedelta(hours=1)
-        min_disk_in, max_disk_in                                        \
-            = ssample_repo.getMinAndMaxDiskInLaterThan(
-                        pred.last_used_server,
-                        last_hour)
-
-        disk_difference = max_disk_in - min_disk_in
-        ram_cached = ssample_repo.getRamCachedWithDiskInGequal(
-                            pred.last_used_server,
-                            max_disk_in)
-
-        # See if cache is still available
-        cache_available = cacheAvailable(
-                disk_difference, pred.avg_ram, ram_cached)
-
-        # if there is still cache available,
-        # we check if the server has enough memory CPU
-        # to support the user
-        if cache_available:
-            server_cpu, latest_time =                                  \
-                    ssample_repo.getLatestServerCPUAndTime(
-                        pred.last_used_server)
-
-            sum_avg_cpu =                                               \
-                pred_repo.getSumAvgCPUFromServerLaterThan(
+    # if not exist user
+    if not pred_repo.exist(uid):
+        return leastLoadServer(server_repo,
+                               ssample_repo, pred_repo)
+
+    # Prediction Record of UID
+    pred = pred_repo.get(uid)
+    last_hour = pred.last_login - datetime.timedelta(hours=1)
+
+    min_disk_in, max_disk_in                                        \
+        = ssample_repo.getMinAndMaxDiskInLaterThan(
                     pred.last_used_server,
-                    latest_time - datetime.timedelta(hours=1))
-
-            if server_cpu < sum_avg_cpu:
-                server_cpu = sum_avg_cpu
-
-            total_cpu = server_cpu + pred.avg_cpu
-            if total_cpu < 100:
-                print(pred.last_used_server)
-            else:
-                leastLoadServer(server_repo, ssample_repo, pred_repo)
-        else:
-            leastLoadServer(server_repo, ssample_repo, pred_repo)
+                    last_hour)
+
+    disk_difference = max_disk_in - min_disk_in
+    ram_cached = ssample_repo.getRamCachedWithDiskInGequal(
+                        pred.last_used_server,
+                        max_disk_in)
+    ram_total = ssample_repo.getServerTotalRam(pred.last_used_server)
+
+    # See if cache is still available
+    cache_available = cacheAvailable(
+        disk_difference,
+        pred.avg_ram / 100 * ram_total,
+        ram_cached
+    )
+
+    if not cache_available:
+        return leastLoadServer(server_repo,
+                               ssample_repo, pred_repo)
+
+    server_cpu, latest_time =                                  \
+        ssample_repo.getLatestServerCPUAndTime(
+            pred.last_used_server
+        )
+
+    sum_avg_cpu =                                               \
+        pred_repo.getSumAvgCPUFromServerLaterThan(
+            pred.last_used_server,
+            latest_time - datetime.timedelta(hours=1)
+        )
+
+    if server_cpu < sum_avg_cpu:
+        server_cpu = sum_avg_cpu
+
+    total_cpu = server_cpu + pred.avg_cpu
+    if total_cpu < 100:
+        return pred.last_used_server
     else:
         leastLoadServer(server_repo, ssample_repo, pred_repo)
 
 
-def cacheAvailable(disk_diff, user_avg_ram, cache):
+def cacheAvailable(disk_diff, user_avg_ram_in_MB, cache):
     cache_not_replace = cache - disk_diff
-    if cache_not_replace > user_avg_ram:
+    if cache_not_replace > user_avg_ram_in_MB:
         return True
     else:
         return False
@@ -94,4 +100,4 @@ def cacheAvailable(disk_diff, user_avg_ram, cache):
 
     uid = int(sys.argv[1])
 
-    loadBalance(uid, server_repo, ssample_repo, pred_repo)
+    print(loadBalance(uid, server_repo, ssample_repo, pred_repo))

monitor.sql

@@ -21,8 +21,8 @@ CREATE TABLE IF NOT EXISTS `MONITOR`.`USER` (
 	CONSTRAINT `USER_SERVER_FK`
 		FOREIGN KEY (`SERVER`)
 		REFERENCES `MONITOR`.`SERVER` (`NAME`)
-	ON DELETE NO ACTION
-	ON UPDATE NO ACTION)
+	ON DELETE CASCADE
+	ON UPDATE CASCADE)
 ENGINE = InnoDB;
 
 
@@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS `MONITOR`.`JOB` (
 	`UID` INT NULL,
 	`START_TIME` DATETIME NULL,
 	`CMD_NAME` VARCHAR(45) NULL,
-	`COMMAND` TINYTEXT NULL,
+	`COMMAND` LONGTEXT NULL,
 	`SERVER` VARCHAR(45) NULL,
 	PRIMARY KEY (`id`) ,
 	UNIQUE INDEX `P_IDX` (`PID` ASC, `UID` ASC, `START_TIME` ASC) ,
@@ -41,13 +41,13 @@ CREATE TABLE IF NOT EXISTS `MONITOR`.`JOB` (
 	CONSTRAINT `JOB_UID_FK`
 		FOREIGN KEY (`UID`)
 		REFERENCES `MONITOR`.`USER` (`UID`)
-		ON DELETE NO ACTION
-		ON UPDATE NO ACTION,
+		ON DELETE CASCADE
+		ON UPDATE CASCADE,
 	CONSTRAINT `JOB_SERVER_FK`
 		FOREIGN KEY (`SERVER`)
 		REFERENCES `MONITOR`.`SERVER` (`NAME`)
-		ON DELETE NO ACTION
-		ON UPDATE NO ACTION)
+		ON DELETE CASCADE
+		ON UPDATE CASCADE)
 ENGINE = InnoDB;
 
 
@@ -68,8 +68,8 @@ CREATE TABLE IF NOT EXISTS `MONITOR`.`jSAMPLE` (
 	CONSTRAINT `P_FK`
 		FOREIGN KEY (`PID`, `UID` , `START_TIME`)
 		REFERENCES `MONITOR`.`JOB` (`PID`, `UID`, `START_TIME`)
-		ON DELETE NO ACTION
-		ON UPDATE NO ACTION)
+		ON DELETE CASCADE
+		ON UPDATE CASCADE)
 ENGINE = InnoDB;
 
 
@@ -81,15 +81,16 @@ CREATE TABLE IF NOT EXISTS `MONITOR`.`sSAMPLE` (
 	`RAM` DECIMAL(6, 3) NULL,
 	`RAM_AVAILABLE` INT NULL,
 	`RAM_CACHED` INT NULL,
+	`RAM_TOTAL` INT NULL,
 	`DISK_IN` INT NULL,
 	`DISK_OUT` INT NULL,
 	PRIMARY KEY (`id`) ,
 	INDEX `SERVER_FK_IDX` (`NAME` ASC),
 	CONSTRAINT `sSAMPLE_SERVER_FK`
 		FOREIGN KEY (`NAME`)
 		REFERENCES `MONITOR`.`SERVER` (`NAME`)
-		ON DELETE NO ACTION
-		ON UPDATE NO ACTION)
+		ON DELETE CASCADE
+		ON UPDATE CASCADE)
 ENGINE = InnoDB;
 
 CREATE TABLE IF NOT EXISTS `MONITOR`.`PREDICTION` (

process.log

@@ -1,23 +1 @@
 
-ERROR:root:Error: 
-Traceback (most recent call last):
-  File "process_monitor.py", line 31, in <module>
-    processes = ProcessService.getAll(excludeProcess, minUID=1000)
-  File "/home/tung/Desktop/ktlt/utils.py", line 96, in getAll
-    p = Process(proc)
-  File "/home/tung/Desktop/ktlt/utils.py", line 38, in __init__
-    self.__getProcessResources(process)
-  File "/home/tung/Desktop/ktlt/utils.py", line 42, in __getProcessResources
-    = process.io_counters()
-  File "/usr/local/lib/python2.7/dist-packages/psutil/__init__.py", line 787, in io_counters
-    return self._proc.io_counters()
-  File "/usr/local/lib/python2.7/dist-packages/psutil/_pslinux.py", line 1322, in wrapper
-    raise AccessDenied(self.pid, self._name)
-AccessDenied: psutil.AccessDenied (pid=1497, name='(sd-pam)')
-ERROR:root:Error: 
-Traceback (most recent call last):
-  File "process_monitor.py", line 40, in <module>
-    jsampleRepo.add(jSample(proc))
-  File "/home/tung/Desktop/ktlt/utils.py", line 331, in add
-    self.db.commit()
-KeyboardInterrupt

process_monitor.py

@@ -7,7 +7,7 @@
 
 if __name__ == '__main__':
     try:
-        db = MySQLdb.connect(host="10.0.2.2", user="lb",
+        db = MySQLdb.connect(host="localhost", user="lb",
                              passwd="lb", db="MONITOR")
     except Exception as e:
         print("Can't connect to DB")

server_monitor.py

@@ -8,7 +8,7 @@
 if __name__ == '__main__':
     logging.basicConfig(level=logging.DEBUG, filename="server.log")
     try:
-        db = MySQLdb.connect(host="10.0.2.2", user="lb",
+        db = MySQLdb.connect(host="localhost", user="lb",
                              passwd="lb", db="MONITOR")
     except Exception as e:
         print("Can't connect to database")

utils.py

@@ -303,10 +303,10 @@ def __init__(self, proc):
 
 class jSampleStatistic:
     def __init__(self, avg_cpu, max_cpu, avg_ram, max_ram, run_time):
-        self.avg_cpu = avg_cpu
-        self.max_cpu = max_cpu
-        self.avg_ram = avg_ram
-        self.max_ram = max_ram
+        self.avg_cpu = round(avg_cpu, 3)
+        self.max_cpu = round(max_cpu, 3)
+        self.avg_ram = round(avg_ram, 3)
+        self.max_ram = round(max_ram, 3)
         self.run_time = run_time
 
 
@@ -361,12 +361,14 @@ def deleteUidsEarlierThan(self, uid, time):
 ##############################
 class sSample:
     def __init__(self, name, cpu, ram,
-                 ram_available, ram_cached, disk_in, disk_out):
+                 ram_available, ram_cached, ram_total,
+                 disk_in, disk_out):
         self.name = name
         self.cpu = cpu
         self.ram = ram
         self.ram_available = ram_available
         self.ram_cached = ram_cached
+        self.ram_total = ram_total
         self.disk_in = disk_in
         self.disk_out = disk_out
 
@@ -383,6 +385,7 @@ def getServerInfo():
         ram = round(percent, 3)
         ram_available = bytesToMegabytes(available)
         ram_cached = bytesToMegabytes(cache)
+        ram_total = bytesToMegabytes(total)
 
         # IO usage
         read_count, write_count, read_bytes, write_bytes,                   \
@@ -394,7 +397,7 @@ def getServerInfo():
 
         hostname = socket.gethostname()
         return sSample(hostname, cpu, ram, ram_available,
-                       ram_cached, disk_in, disk_out)
+                       ram_cached, ram_total, disk_in, disk_out)
 
 
 class sSampleRepository:
@@ -405,11 +408,12 @@ def add(self, ssample):
         cursor = self.db.cursor()
         try:
             cursor.execute("INSERT INTO sSAMPLE(NAME, CPU, RAM, "
-                           "RAM_AVAILABLE, RAM_CACHED, DISK_IN, "
-                           "DISK_OUT) VALUES"
-                           "(%s, %s, %s, %s, %s, %s, %s)",
+                           "RAM_AVAILABLE, RAM_CACHED, RAM_TOTAL, "
+                           "DISK_IN, DISK_OUT) VALUES"
+                           "(%s, %s, %s, %s, %s, %s, %s, %s)",
                            (ssample.name, ssample.cpu, ssample.ram,
-                            ssample.ram_available, ssample.ram_cached,
+                            ssample.ram_available,
+                            ssample.ram_cached, ssample.ram_total,
                             ssample.disk_in, ssample.disk_out))
             self.db.commit()
             cursor.close()
@@ -463,7 +467,19 @@ def getLatestServerCPUAndTime(self, server_name):
         except Exception as e:
             print("sSampleRepository: getLatestServerCPUAndTime()")
             print(e)
-        return cpu, timestamp
+        return float(cpu), timestamp
+
+    def getServerTotalRam(self, server_name):
+        cursor = self.db.cursor()
+        try:
+            cursor.execute("SELECT RAM_TOTAL FROM sSAMPLE "
+                           "WHERE NAME = %s", (server_name, ))
+            for row in cursor:
+                return row[0]
+        except Exception as e:
+            print("sSampleRepository: getServerTotalRam()")
+            print(e)
+        return None
 
 
 ##############################
@@ -570,4 +586,4 @@ def getSumAvgCPUFromServerLaterThan(self, server_name, timestamp):
         except Exception as e:
             print("PredictionRepository: getListAvgCPUFromServerLaterThan()")
             print(e)
-        return sum(avgs_cpu)
+        return float(sum(avgs_cpu))