basic mining

Author: ckrause

loda/mine/miner.py

@@ -1,5 +1,9 @@
 """Miner class for finding integer sequence programs."""
 
+import os
+import time
+
+from loda.lang import Program
 from loda.oeis import PrefixIndex, Sequence
 from loda.runtime import Evaluator, Interpreter
 
@@ -10,22 +14,52 @@ def __init__(self, sequences: list, interpreter: Interpreter, generator):
         self.__index = PrefixIndex(sequences)
         self.__interpreter = interpreter
         self.__generator = generator
+        self.__basic_timeout = 60
+        self.__extended_timeout = 600
 
     def __call__(self):
         program = self.__generator()
         evaluator = Evaluator(program, self.__interpreter)
         match = self.__index.global_match()
         refine = True
+        start_time = time.time()
         try:
             while refine:
                 term = evaluator()
-                # print(term)
                 refine = self.__index.refine_match(match, term)
+                if (time.time() - start_time) > self.__basic_timeout:
+                    print("timeout")
+                    return
         except Exception as e:
             print("evaluation error: {}".format(e))
             return
-
         ids = self.__index.get_match_ids(match)
         for id in ids:
-            print("Found match for {}".format(Sequence(id)))
-            print(program)
+            self.__check_match(program, id)
+
+    def __check_match(self, program: Program, id: int):
+        seq = self.__index.get(id)
+        terms = seq.load_b_file(os.path.expanduser("~/loda/oeis"))
+        if terms is None or len(terms) == len(seq.terms):
+            print("Skipping check for {}".format(seq))
+            return
+        print("Checking match for {}".format(seq))
+        start_time = time.time()
+        if len(terms) > 1000:
+            terms = terms[0:1000]
+        evaluator = Evaluator(program, self.__interpreter)
+        correct = True
+        for t in terms:
+            n = evaluator()
+            if n != t:
+                correct = False
+                break
+            if (time.time() - start_time) > self.__extended_timeout:
+                print("timeout")
+                return
+        if correct:
+            print("Found match for {}".format(seq))
+            p = "~/loda/programs/local/{}.asm".format(seq.id_str())
+            with open(os.path.expanduser(p), "w") as asm_file:
+                asm_file.write("; {}\n".format(seq))
+                asm_file.write(str(program))

loda/oeis/sequence.py

@@ -3,6 +3,8 @@
 import functools
 import os.path
 import re
+import requests
+import subprocess
 
 
 @functools.total_ordering
@@ -86,10 +88,18 @@ def load_b_file(self, path: str) -> list:
                 folder that contains the b-files in sub-directories, e.g. `b/123/b123456.txt`.
         """
         terms = []
+        txt = "b{:06}.txt".format(self.id)
         if len(path) == 0 or os.path.isdir(path):
             dir = "{:03}".format(self.id//1000)
-            txt = "b{:06}.txt".format(self.id)
             path = os.path.join(path, "b", dir, txt)
+        if not os.path.isfile(path):
+            b_url = "http://api.loda-lang.org/miner/v1/oeis/{}.gz".format(txt)
+            print("Fetching {}".format(b_url))
+            req = requests.get(b_url)
+            gz_path = path + ".gz"
+            with open(gz_path, 'wb') as gz:
+                gz.write(req.content)
+            subprocess.run(["gunzip", gz_path])
         with open(path) as b_file:
             expected_index = -1
             for line in b_file:

mine.py

@@ -0,0 +1,34 @@
+import datetime
+import os.path
+
+from loda.oeis import ProgramCache, Sequence
+from loda.runtime import Interpreter
+from loda.mine import Miner
+from loda.ml.keras.program_generation_rnn import load_model, Generator
+
+
+def mine(model_path: str):
+
+    model = load_model(model_path)
+    model.summary()
+    generator = Generator(model, num_lanes=10)
+    programs_dir = os.path.expanduser("~/loda/programs/oeis")
+    program_cache = ProgramCache(programs_dir)
+    existing_ids = set(program_cache.all_ids())
+    seqs = Sequence.load_oeis(os.path.expanduser("~/loda/oeis"))
+    seqs = list(filter(lambda s: len(s.terms) >=
+                8 and s.id not in existing_ids, seqs))
+    print("Loaded {} sequences".format(len(seqs)))
+    interpreter = Interpreter(program_cache)
+    miner = Miner(seqs, interpreter, generator)
+    i = 0
+    while True:
+        miner()
+        i += 1
+        if i % 100 == 0:
+            ct = datetime.datetime.now()
+            print(ct, generator.get_stats_info_str())
+
+
+if __name__ == "__main__":
+    mine(os.path.expanduser("~/scripts/model-001"))

requirements.txt

@@ -1,4 +1,5 @@
 nose2
 parameterized
+requests
 tensorflow; sys_platform != 'darwin'
 tensorflow-macos; sys_platform == 'darwin'

train.py

@@ -0,0 +1,23 @@
+import os.path
+
+from loda.oeis import ProgramCache
+from loda.ml.keras.program_generation_rnn import train_model
+
+
+def train(programs_percentage: int):
+    programs_dir = os.path.expanduser("~/loda/programs/oeis")
+    program_cache = ProgramCache(programs_dir)
+    num_train_programs = -1
+    if programs_percentage < 100:
+        num_total_programs = len(program_cache.all_ids())
+        num_train_programs = (programs_percentage * num_total_programs) // 100
+    print("Training using {} programs".format(num_train_programs))
+    model = train_model(program_cache, num_programs=num_train_programs)
+    model.save("model-{:03}".format(programs_percentage))
+
+
+if __name__ == "__main__":
+    train(1)
+    train(25)
+    train(50)
+    train(100)