Merge pull request #46 from snowyu/master

fix: RuntimeError: can't start new thread

Author: EvilCult

python/plugins/dotpy.py

@@ -6,6 +6,7 @@
 import re
 import db
 import threading
+from .threads import ThreadPool
 
 class Source (object) :
 
@@ -18,19 +19,22 @@ def getSource (self) :
         with open(sourcePath, 'r') as f:
             lines = f.readlines()
             total = len(lines)
-            threads = []
+            threads = ThreadPool(20)
 
             for i in range(0, total):
                 line = lines[i].strip('\n')
                 item = line.split(',', 1)
-                thread = threading.Thread(target = self.detectData, args = (item[0], item[1], ), daemon = True)
-                thread.start()
-                threads.append(thread)
+                threads.add_task(self.detectData, title = item[0], url = item[1])
+                # thread = threading.Thread(target = self.detectData, args = (item[0], item[1], ), daemon = True)
+                # thread.start()
+                # threads.append(thread)
+            threads.wait_completion()
 
-            for t in threads:
-                t.join()
+            # for t in threads:
+            #     t.join()
 
     def detectData (self, title, url) :
+        print('detectData', title, url)
         info = self.T.fmtTitle(title)
 
         netstat = self.T.chkPlayable(url)
@@ -62,4 +66,4 @@ def addData (self, data) :
             DB.insert(data)
         else :
             id = result[0][0]
-            DB.edit(id, data)
+            DB.edit(id, data)

python/plugins/threads.py

@@ -0,0 +1,54 @@
+from queue import Queue
+from threading import Thread
+
+
+class Worker(Thread):
+    """Thread executing tasks from a given tasks queue"""
+    def __init__(self, tasks):
+        Thread.__init__(self)
+        self.tasks = tasks
+        self.daemon = True
+        self.start()
+
+    def run(self):
+        while True:
+            func, args, kargs = self.tasks.get()
+            try:
+                func(*args, **kargs)
+            except Exception as e:
+                print(e)
+            finally:
+                self.tasks.task_done()
+
+
+class ThreadPool:
+    """Pool of threads consuming tasks from a queue"""
+    def __init__(self, num_threads):
+        self.tasks = Queue(num_threads)
+        for _ in range(num_threads):
+            Worker(self.tasks)
+
+    def add_task(self, func, *args, **kargs):
+        """Add a task to the queue"""
+        self.tasks.put((func, args, kargs))
+
+    def wait_completion(self):
+        """Wait for completion of all the tasks in the queue"""
+        self.tasks.join()
+
+# if __name__ == '__main__':
+#     from random import randrange
+#     from time import sleep
+
+#     delays = [randrange(1, 10) for i in range(100)]
+
+#     def wait_delay(d):
+#         print 'sleeping for (%d)sec' % d
+#         sleep(d)
+
+#     pool = ThreadPool(20)
+
+#     for i, d in enumerate(delays):
+#         pool.add_task(wait_delay, d)
+
+#     pool.wait_completion()