From d3e708e7af98bcf58af54d5eaf1fdf02c165c8dd Mon Sep 17 00:00:00 2001
From: Karsten Schmidt <k@thi.ng>
Date: Tue, 12 Sep 2023 16:13:01 +0200
Subject: [PATCH] feat(sexpr): add support for line comments, fix tokenize

- update SyntaxOpts
- fix/update tokenize() to emit
---
 packages/sexpr/src/api.ts      |  6 ++++++
 packages/sexpr/src/tokenize.ts |  9 ++++++++-
 packages/sexpr/test/index.ts   | 36 ++++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/packages/sexpr/src/api.ts b/packages/sexpr/src/api.ts
index e15fe57e91..5fe79ba47d 100644
--- a/packages/sexpr/src/api.ts
+++ b/packages/sexpr/src/api.ts
@@ -19,6 +19,11 @@ export interface SyntaxOpts {
 	 * Default: `"`
 	 */
 	string: string;
+	/**
+	 * Single character string for comments (always active until next line break)
+	 * Default: `;`
+	 */
+	comment: string;
 }
 
 export interface Token {
@@ -107,4 +112,5 @@ export const DEFAULT_SYNTAX: SyntaxOpts = {
 	scopes: [["(", ")"]],
 	whiteSpace: /(\s|,)/,
 	string: '"',
+	comment: ";",
 };
diff --git a/packages/sexpr/src/tokenize.ts b/packages/sexpr/src/tokenize.ts
index bf097ef426..4d96249261 100644
--- a/packages/sexpr/src/tokenize.ts
+++ b/packages/sexpr/src/tokenize.ts
@@ -20,6 +20,7 @@ export function* tokenize(
 		scopes: rawScopes,
 		whiteSpace,
 		string,
+		comment,
 	} = {
 		...DEFAULT_SYNTAX,
 		...opts,
@@ -29,6 +30,7 @@ export function* tokenize(
 		.join("");
 	let token = "";
 	let isString = false;
+	let isComment = false;
 	let tokenLine = 0;
 	let tokenCol = 0;
 	let line = 0;
@@ -45,7 +47,9 @@ export function* tokenize(
 		} else {
 			col++;
 		}
-		if (!isString) {
+		if (isComment) {
+			if (c === "\n") isComment = false;
+		} else if (!isString) {
 			if (whiteSpace.test(c)) {
 				token && (yield $(token));
 				token = "";
@@ -62,6 +66,8 @@ export function* tokenize(
 				tokenCol = col;
 				token = '"';
 				isString = true;
+			} else if (c === comment) {
+				isComment = true;
 			} else {
 				if (!token) {
 					tokenLine = line;
@@ -78,4 +84,5 @@ export function* tokenize(
 			token += c;
 		}
 	}
+	token && (yield $(token));
 }
diff --git a/packages/sexpr/test/index.ts b/packages/sexpr/test/index.ts
index 7ef7458710..b4df74bdb2 100644
--- a/packages/sexpr/test/index.ts
+++ b/packages/sexpr/test/index.ts
@@ -70,6 +70,17 @@ group("sexpr", {
 		});
 	},
 
+	"non-expression root": () => {
+		assert.deepStrictEqual(parse("x"), {
+			type: "root",
+			children: [{ type: "sym", value: "x" }],
+		});
+		assert.deepStrictEqual(parse("23"), {
+			type: "root",
+			children: [{ type: "num", value: 23 }],
+		});
+	},
+
 	"custom syntax": () => {
 		const syntax: Partial<SyntaxOpts> = {
 			scopes: [
@@ -149,4 +160,29 @@ group("sexpr", {
 	"missing fn in env": () => {
 		assert.throws(() => $eval("(foo)"));
 	},
+
+	"line comment": () => {
+		assert.deepStrictEqual(
+			parse(`
+; intro
+(def x ; ignore me
+; line 2
+  ; line 3
+23)`),
+			{
+				type: "root",
+				children: [
+					{
+						type: "expr",
+						value: "(",
+						children: [
+							{ type: "sym", value: "def" },
+							{ type: "sym", value: "x" },
+							{ type: "num", value: 23 },
+						],
+					},
+				],
+			}
+		);
+	},
 });