Skip to content

Commit

Permalink
Add TeX syntax highlighting
Browse files Browse the repository at this point in the history
  • Loading branch information
jart committed Oct 26, 2024
1 parent 5ead373 commit 6260c7b
Show file tree
Hide file tree
Showing 5 changed files with 236 additions and 6 deletions.
4 changes: 4 additions & 0 deletions llamafile/highlight.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ Highlight *Highlight::create(const std::string_view &lang) {
lang == "rb")
return new HighlightRuby;

if (lang == "tex" || //
lang == "latex")
return new HighlightTex;

if (lang == "fs" || //
lang == "4th" || //
lang == "frt" || //
Expand Down
16 changes: 15 additions & 1 deletion llamafile/highlight.h
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@
#define HI_QUALIFIER "\033[35m" // magenta
#define HI_IMMEDIATE "\033[36m" // cyan
#define HI_REGISTER "\033[1;35m" // bold magenta
#define HI_DATE "\033[33m" // yellow
#define HI_DIRECTIVE "\033[1;31m" // bold red
#define HI_WARNING "\033[1;31m" // bold red
#define HI_MATH "\033[1;35m" // bold magenta

typedef const char *is_keyword_f(const char *, size_t);

Expand Down Expand Up @@ -593,3 +593,17 @@ class HighlightLd : public Highlight {
bool cpp_ = false;
std::string word_;
};

class HighlightTex : public Highlight {
public:
HighlightTex();
~HighlightTex() override;
void feed(std::string *result, std::string_view input) override;
void flush(std::string *result) override;

private:
int c_ = 0;
int u_ = 0;
int t_ = 0;
std::string word_;
};
10 changes: 5 additions & 5 deletions llamafile/highlight_tcl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,18 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {
} else if (c == '"') {
t_ = DQUOTE;
*r += HI_STRING;
append_wchar(r, c);
*r += '"';
} else if (c == '$') {
append_wchar(r, c);
*r += '$';
t_ = VAR;
} else if (c == '#') {
*r += HI_COMMENT;
append_wchar(r, c);
*r += '#';
t_ = COMMENT;
} else if (c == '\\') {
t_ = BACKSLASH;
*r += HI_ESCAPE;
append_wchar(r, c);
*r += '\\';
} else {
append_wchar(r, c);
}
Expand Down Expand Up @@ -115,7 +115,7 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {

case VAR:
if (c == '{') {
append_wchar(r, c);
*r += '{';
*r += HI_VAR;
t_ = VAR_CURLY;
break;
Expand Down
1 change: 1 addition & 0 deletions llamafile/highlight_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ const char *const kLanguages[] = {
"sql", //
"swift", //
"tcl", //
"tex", //
"txt", //
"typescript", //
"zig", //
Expand Down
211 changes: 211 additions & 0 deletions llamafile/highlight_tex.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
//
// Copyright 2024 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "highlight.h"
#include "string.h"
#include <cosmo.h>
#include <ctype.h>

enum {
NORMAL,
BACKSLASH,
COMMAND,
COMMENT,
DOLLAR,
MATH,
MATH_BACKSLASH,
BACKTICK,
STRING,
STRING_QUOTE,
};

HighlightTex::HighlightTex() {
}

HighlightTex::~HighlightTex() {
}

void HighlightTex::feed(std::string *r, std::string_view input) {
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
if (!u_) {
if (b < 0300) {
c = b;
} else {
c_ = ThomPikeByte(b);
u_ = ThomPikeLen(b) - 1;
continue;
}
} else {
c = c_ = ThomPikeMerge(c_, b);
if (--u_)
continue;
}
switch (t_) {

Normal:
case NORMAL:
if (c == '\\') {
t_ = BACKSLASH;
} else if (c == '$') {
t_ = DOLLAR;
} else if (c == '`') {
t_ = BACKTICK;
} else if (c == '%') {
t_ = COMMENT;
*r += HI_COMMENT;
*r += '%';
} else {
append_wchar(r, c);
}
break;

case BACKSLASH:
if (c == '\\') {
*r += HI_WARNING;
*r += "\\\\";
*r += HI_RESET;
t_ = NORMAL;
} else if (isspace(c)) {
*r += '\\';
t_ = NORMAL;
goto Normal;
} else if (isalpha(c) || c == '@') {
*r += HI_KEYWORD;
*r += '\\';
append_wchar(r, c);
t_ = COMMAND;
} else {
*r += HI_ESCAPE;
*r += '\\';
append_wchar(r, c);
*r += HI_RESET;
t_ = NORMAL;
}
break;

case COMMAND:
if (isalpha(c) || c == '@') {
append_wchar(r, c);
} else {
*r += HI_RESET;
t_ = NORMAL;
goto Normal;
}
break;

case DOLLAR:
if (c == '$') {
*r += "$$";
t_ = NORMAL;
} else if (c == '\\') {
*r += HI_MATH;
*r += "$\\";
t_ = MATH_BACKSLASH;
} else {
*r += HI_MATH;
*r += "$";
append_wchar(r, c);
t_ = MATH;
}
break;

case MATH:
if (c == '$') {
*r += "$";
*r += HI_RESET;
t_ = NORMAL;
} else if (c == '\\') {
*r += '\\';
t_ = MATH_BACKSLASH;
} else {
append_wchar(r, c);
}
break;

case MATH_BACKSLASH:
append_wchar(r, c);
t_ = MATH;
break;

case COMMENT:
append_wchar(r, c);
if (c == '\n') {
*r += HI_RESET;
t_ = NORMAL;
}
break;

case BACKTICK:
if (c == '`') {
*r += HI_STRING;
*r += "``";
t_ = STRING;
} else {
*r += '`';
t_ = NORMAL;
goto Normal;
}
break;

case STRING:
append_wchar(r, c);
if (c == '\'')
t_ = STRING_QUOTE;
break;

case STRING_QUOTE:
append_wchar(r, c);
if (c == '\'') {
*r += HI_RESET;
t_ = NORMAL;
} else {
t_ = STRING;
}
break;

default:
__builtin_unreachable();
}
}
}

void HighlightTex::flush(std::string *r) {
switch (t_) {
case BACKTICK:
*r += '`';
break;
case DOLLAR:
*r += '$';
break;
case BACKSLASH:
*r += '\\';
break;
case COMMAND:
case COMMENT:
case MATH:
case MATH_BACKSLASH:
case STRING:
case STRING_QUOTE:
*r += HI_RESET;
break;
default:
break;
}
t_ = NORMAL;
}

0 comments on commit 6260c7b

Please sign in to comment.