From dcd31ea9db16e7144b328e34276ec91344ffac8d Mon Sep 17 00:00:00 2001 From: Andy Lamperski Date: Mon, 23 Jul 2018 14:48:08 -0500 Subject: [PATCH 1/8] fixed handling of non-executed cells in jupyter notebooks --- pweave/formatters/jupyter_notebook.py | 9 +++++++-- pweave/processors/base.py | 2 +- pweave/pweb.py | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pweave/formatters/jupyter_notebook.py b/pweave/formatters/jupyter_notebook.py index ae96b44..8a6862f 100644 --- a/pweave/formatters/jupyter_notebook.py +++ b/pweave/formatters/jupyter_notebook.py @@ -47,10 +47,14 @@ def format(self): } ) if chunk["type"] == "code": + if chunk["evaluate"]: + ec = self.execution_count + else: + ec = None self.notebook["cells"].append( { "cell_type": "code", - "execution_count" : self.execution_count, + "execution_count" : ec, "metadata": { "collapsed": False, "autoscroll": "auto", @@ -60,7 +64,8 @@ def format(self): "outputs" : chunk["result"] } ) - self.execution_count +=1 + if chunk['evaluate']: + self.execution_count +=1 self.notebook = nbformat.from_dict(self.notebook) def getformatted(self): diff --git a/pweave/processors/base.py b/pweave/processors/base.py index df5e4d0..231b524 100644 --- a/pweave/processors/base.py +++ b/pweave/processors/base.py @@ -138,7 +138,7 @@ def _runcode(self, chunk): self.pending_code = "" if not chunk['evaluate']: - chunk['result'] = '' + chunk['result'] = "" return chunk self.pre_run_hook(chunk) diff --git a/pweave/pweb.py b/pweave/pweb.py index 6f294b6..7c89fc1 100644 --- a/pweave/pweb.py +++ b/pweave/pweb.py @@ -29,6 +29,7 @@ class Pweb(object): def __init__(self, source, doctype = None, *, informat = None, kernel = "python3", output = None, figdir = 'figures', mimetype = None): + self.source = source name, ext = os.path.splitext(os.path.basename(source)) self.basename = name From 3c5ec277dd12bc1a8de333b1ef09dda2e0ba2643 Mon Sep 17 00:00:00 2001 From: Andy Lamperski Date: Mon, 23 Jul 2018 14:50:43 -0500 Subject: [PATCH 2/8] removed the extra print commands --- pweave/processors/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pweave/processors/base.py b/pweave/processors/base.py index 231b524..1be97ae 100644 --- a/pweave/processors/base.py +++ b/pweave/processors/base.py @@ -138,7 +138,7 @@ def _runcode(self, chunk): self.pending_code = "" if not chunk['evaluate']: - chunk['result'] = "" + chunk['result'] = [] return chunk self.pre_run_hook(chunk) From f884d6659bfb7879044865bec94649d9fab390d4 Mon Sep 17 00:00:00 2001 From: Andy Lamperski Date: Mon, 23 Jul 2018 17:53:13 -0500 Subject: [PATCH 3/8] got the tangle command to take a separe output filename --- pweave/__init__.py | 4 ++-- pweave/pweb.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pweave/__init__.py b/pweave/__init__.py index f7c5766..5fb9d41 100644 --- a/pweave/__init__.py +++ b/pweave/__init__.py @@ -58,12 +58,12 @@ def weave(file, doctype=None, informat=None, kernel="python3", plot=True, doc.weave() -def tangle(file, informat = None): +def tangle(file, informat = None, output = None): """Tangles a noweb file i.e. extracts code from code chunks to a .py file :param file: ``string`` the pweave document containing the code """ - doc = Pweb(file, kernel = None, informat = informat) + doc = Pweb(file, kernel = None, informat = informat, output = output) doc.tangle() diff --git a/pweave/pweb.py b/pweave/pweb.py index 7c89fc1..759ecc1 100644 --- a/pweave/pweb.py +++ b/pweave/pweb.py @@ -198,6 +198,8 @@ def tangle(self): """Tangle the document""" if self.output is None: target = os.path.join(self.wd, self.basename + '.py') + else: + target = self.output code = [x for x in self.parsed if x['type'] == 'code'] main = '\nif __name__ == "__main__":' for x in code: From 936a591406fe753f82ad62f820fbedacd9cd875b Mon Sep 17 00:00:00 2001 From: Andy Lamperski Date: Tue, 24 Jul 2018 09:05:57 -0500 Subject: [PATCH 4/8] enabled boolean options to be specified via inline code --- pweave/formatters/base.py | 2 +- pweave/formatters/jupyter_notebook.py | 4 +++- pweave/processors/base.py | 25 ++++++++++++++++++++++++- pweave/readers.py | 3 +++ 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pweave/formatters/base.py b/pweave/formatters/base.py index acd448d..d0c5786 100644 --- a/pweave/formatters/base.py +++ b/pweave/formatters/base.py @@ -212,7 +212,7 @@ def format_codechunks(self, chunk): if chunk['echo']: chunk["content"] = self.fix_linefeeds(chunk["content"]) result += '%(codestart)s%(content)s%(codeend)s' % chunk - + if chunk['results'] != 'hidden': stream_result = {"output_type" : "stream", "text" : ""} other_result = "" diff --git a/pweave/formatters/jupyter_notebook.py b/pweave/formatters/jupyter_notebook.py index 8a6862f..a19780c 100644 --- a/pweave/formatters/jupyter_notebook.py +++ b/pweave/formatters/jupyter_notebook.py @@ -46,11 +46,13 @@ def format(self): "source": chunk["content"], } ) - if chunk["type"] == "code": + if chunk["type"] == "code" and chunk["echo"]: if chunk["evaluate"]: ec = self.execution_count else: ec = None + + self.notebook["cells"].append( { "cell_type": "code", diff --git a/pweave/processors/base.py b/pweave/processors/base.py index 1be97ae..78ee8ec 100644 --- a/pweave/processors/base.py +++ b/pweave/processors/base.py @@ -96,6 +96,25 @@ def _runcode(self, chunk): # Add defaultoptions to parsed options if chunk['type'] == 'code': + + + for key in chunk["options"].keys(): + opt = chunk["options"][key] + if isinstance(opt,str): + procString = self.loadinline(opt) + + # There is probably a more elegant solution + # This will only work with boolean values + if procString == 'True': + procVal = True + elif procString == 'False': + procVal = False + else: + procVal = procString + + chunk["options"][key] = procVal + + defaults = rcParams["chunk"]["defaultoptions"].copy() defaults.update(chunk["options"]) chunk.update(defaults) @@ -246,7 +265,7 @@ def loadinline(self, content): if not elem.startswith('<%'): continue if elem.startswith('<%='): - code_str = elem.replace('<%=', '').replace('%>', '').lstrip() + code_str = self.get_code_str(elem) result = self.load_inline_string(code_str).strip() splitted[i] = result continue @@ -256,6 +275,10 @@ def loadinline(self, content): splitted[i] = result return ''.join(splitted) + def get_code_str(self,elem): + code_str = elem.replace('<%=', '').replace('%>', '').lstrip() + return code_str + def add_echo(self, code_str): return 'print(%s),' % code_str diff --git a/pweave/readers.py b/pweave/readers.py index dad86ed..4eae7c6 100644 --- a/pweave/readers.py +++ b/pweave/readers.py @@ -128,6 +128,7 @@ def getoptions(self, line): splitted[0] = 'name = "%s"' % splitted[0] optstring = ','.join(splitted) + optstring = optstring.replace("<%","'<%").replace("%>","%>'") opt_scope = {} exec("chunkoptions = dict(" + optstring + ")", opt_scope) chunkoptions = opt_scope["chunkoptions"] @@ -253,6 +254,8 @@ def getoptions(self, line): splitted[0] = 'name = "%s"' % splitted[0] optstring = ','.join(splitted) + + opt_scope = {} exec("chunkoptions = dict(" + optstring + ")", opt_scope) chunkoptions = opt_scope["chunkoptions"] From c65c7da2695c7e9cd40147b8e2cdeb84b025a8ab Mon Sep 17 00:00:00 2001 From: Andy Lamperski Date: Tue, 24 Jul 2018 09:22:21 -0500 Subject: [PATCH 5/8] enabled echo feature in tangled code, and enabled code execution before tangling --- pweave/__init__.py | 4 ++-- pweave/pweb.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pweave/__init__.py b/pweave/__init__.py index 5fb9d41..ceeaf08 100644 --- a/pweave/__init__.py +++ b/pweave/__init__.py @@ -58,12 +58,12 @@ def weave(file, doctype=None, informat=None, kernel="python3", plot=True, doc.weave() -def tangle(file, informat = None, output = None): +def tangle(file, informat = None, kernel = "python3", output = None): """Tangles a noweb file i.e. extracts code from code chunks to a .py file :param file: ``string`` the pweave document containing the code """ - doc = Pweb(file, kernel = None, informat = informat, output = output) + doc = Pweb(file, kernel = kernel, informat = informat, output = output) doc.tangle() diff --git a/pweave/pweb.py b/pweave/pweb.py index 759ecc1..b443bb8 100644 --- a/pweave/pweb.py +++ b/pweave/pweb.py @@ -196,17 +196,26 @@ def weave(self): def tangle(self): """Tangle the document""" + + # Execute what code should be executed before writing + if self.kernel is not None: + self.run() + chunks = self.executed + else: + chunks = self.parsed + if self.output is None: target = os.path.join(self.wd, self.basename + '.py') else: target = self.output - code = [x for x in self.parsed if x['type'] == 'code'] + code = [x for x in chunks if x['type'] == 'code'] main = '\nif __name__ == "__main__":' for x in code: if 'main' in x['options'] and x['options']['main']: x['content'] = x['content'].replace("\n", "\n ") x['content'] = "".join([main, x['content']]) - code = [x['content'] for x in code] + + code = [x['content'] for x in code if x['options']['echo'] == True] f = open(target, 'w') f.write('\n'.join(code) + "\n") f.close() From 4d3aa05a167ab8119efddcb69020f92fa7ab53a8 Mon Sep 17 00:00:00 2001 From: Andy Lamperski Date: Sat, 28 Jul 2018 22:23:37 -0500 Subject: [PATCH 6/8] enabled inline code within code blocks --- pweave/processors/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pweave/processors/base.py b/pweave/processors/base.py index 78ee8ec..eec09ae 100644 --- a/pweave/processors/base.py +++ b/pweave/processors/base.py @@ -144,6 +144,7 @@ def _runcode(self, chunk): if chunk['type'] == 'code': + chunk['content'] = self.loadinline(chunk['content']) sys.stdout.write("Processing chunk %(number)s named %(name)s from line %(start_line)s\n" % chunk) old_content = None From be6f0a0512faa26aaa213ef25ccf4558fe009953 Mon Sep 17 00:00:00 2001 From: Andy Lamperski Date: Mon, 3 Sep 2018 12:49:22 -0500 Subject: [PATCH 7/8] allowed for more general kernals by passing kernelspec to metadata --- pweave/formatters/jupyter_notebook.py | 23 +++++++++++------------ pweave/pweb.py | 9 +++++++-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pweave/formatters/jupyter_notebook.py b/pweave/formatters/jupyter_notebook.py index a19780c..beeed74 100644 --- a/pweave/formatters/jupyter_notebook.py +++ b/pweave/formatters/jupyter_notebook.py @@ -4,21 +4,20 @@ class PwebNotebookFormatter(object): def __init__(self, executed, *, kernel = "python3", language = "python", + kernel_spec = {}, mimetype = "text/markdown", source = None, theme = None, figdir = None, wd = None): - self.notebook = {"metadata" : { - "kernel_info" : { - "name" : kernel - }, - "language_info": { - # if language_info is defined, its name field is required. - "name": language - } - }, - "nbformat": 4, - "nbformat_minor": 0, - "cells": [ ] + self.notebook = {"metadata" : {"kernel_info" : {"name" : kernel}, + "language_info": { + # if language_info is defined, + # its name field is required. + "name": language + }, + "kernelspec" : kernel_spec }, + "nbformat": 4, + "nbformat_minor": 0, + "cells": [ ] } self.execution_count = 1 diff --git a/pweave/pweb.py b/pweave/pweb.py index b443bb8..db4932f 100644 --- a/pweave/pweb.py +++ b/pweave/pweb.py @@ -39,7 +39,8 @@ def __init__(self, source, doctype = None, *, informat = None, kernel = "python3 self.sink = None self.kernel = None self.language = None - + self.kernel_spec = {} + if mimetype is None: self.mimetype = MimeTypes.guess_mimetype(self.source) else: @@ -81,7 +82,10 @@ def setkernel(self, kernel): """Set the kernel for jupyter_client""" self.kernel = kernel if kernel is not None: - self.language = kernelspec.get_kernel_spec(kernel).language + ks = kernelspec.get_kernel_spec(kernel) + self.language = ks.language + self.kernel_spec = ks.to_dict() + self.kernel_spec['name'] = kernel def getformat(self): """Get current format dictionary. See: http://mpastell.com/pweave/customizing.html""" @@ -150,6 +154,7 @@ def setformat(self, doctype = None, Formatter = None): self.formatter = Formatter([], kernel = self.kernel, language = self.language, + kernel_spec = self.kernel_spec, mimetype = self.mimetype.type, source = self.source, theme = self.theme, From 76860bfa8e965f15e9b498bfd2acf54527214065 Mon Sep 17 00:00:00 2001 From: Andy Lamperski Date: Tue, 4 Sep 2018 08:12:15 -0500 Subject: [PATCH 8/8] fixed the kernelspec on the base pwebformatter --- pweave/formatters/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pweave/formatters/base.py b/pweave/formatters/base.py index d0c5786..0126d7c 100644 --- a/pweave/formatters/base.py +++ b/pweave/formatters/base.py @@ -9,6 +9,7 @@ class PwebFormatter(object): """Base class for all not-notebook formatters""" def __init__(self, executed, *, kernel = "python3", language = "python", + kernel_spec = {}, mimetype = None, source = None, theme = None, figdir = "figures", wd = "."):