feat(muscript): implement parsing
This commit is contained in:
parent
1a210837f3
commit
55b3b4cbe0
GPG Key ID:
E76429612C2929F4
|
|
@ -0,0 +1,29 @@
|
|||
import io.gitlab.jfronny.scripts.*
|
||||
|
||||
plugins {
|
||||
commons.library
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation(projects.commons)
|
||||
api(projects.muscriptAst)
|
||||
|
||||
testImplementation(libs.junit.jupiter.api)
|
||||
testRuntimeOnly(libs.junit.jupiter.engine)
|
||||
}
|
||||
|
||||
publishing {
|
||||
publications {
|
||||
create<MavenPublication>("maven") {
|
||||
groupId = "io.gitlab.jfronny"
|
||||
artifactId = "muscript-parser"
|
||||
|
||||
from(components["java"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.javadoc {
|
||||
linksOffline("https://maven.frohnmeyer-wds.de/javadoc/artifacts/io/gitlab/jfronny/commons/$version/raw", projects.commons)
|
||||
linksOffline("https://maven.frohnmeyer-wds.de/javadoc/artifacts/io/gitlab/jfronny/muscript-ast/$version/raw", projects.muscriptAst)
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package io.gitlab.jfronny.muscript.parser;
|
||||
|
||||
public enum MuScriptVersion {
|
||||
V1, V2, V3;
|
||||
|
||||
public static final MuScriptVersion DEFAULT = V3;
|
||||
|
||||
public boolean contains(MuScriptVersion version) {
|
||||
return compareTo(version) >= 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,509 @@
|
|||
package io.gitlab.jfronny.muscript.parser;
|
||||
|
||||
import io.gitlab.jfronny.muscript.ast.*;
|
||||
import io.gitlab.jfronny.muscript.ast.bool.And;
|
||||
import io.gitlab.jfronny.muscript.ast.bool.Not;
|
||||
import io.gitlab.jfronny.muscript.ast.bool.Or;
|
||||
import io.gitlab.jfronny.muscript.ast.context.ExprUtils;
|
||||
import io.gitlab.jfronny.muscript.ast.context.Script;
|
||||
import io.gitlab.jfronny.muscript.ast.context.TypeMismatchException;
|
||||
import io.gitlab.jfronny.muscript.ast.dynamic.*;
|
||||
import io.gitlab.jfronny.muscript.ast.number.*;
|
||||
import io.gitlab.jfronny.muscript.ast.string.Concatenate;
|
||||
import io.gitlab.jfronny.muscript.core.CodeLocation;
|
||||
import io.gitlab.jfronny.muscript.parser.lexer.LegacyLexer;
|
||||
import io.gitlab.jfronny.muscript.parser.lexer.Lexer;
|
||||
import io.gitlab.jfronny.muscript.parser.lexer.LexerImpl;
|
||||
import io.gitlab.jfronny.muscript.parser.lexer.Token;
|
||||
import io.gitlab.jfronny.muscript.parser.util.PrettyPrintError;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class Parser extends VersionedComponent {
|
||||
private final Lexer lexer;
|
||||
|
||||
private Lexer.Token previous = null;
|
||||
|
||||
public static Expr parse(MuScriptVersion version, String source) {
|
||||
return parse(version, source, null);
|
||||
}
|
||||
|
||||
public static Expr parse(MuScriptVersion version, String source, String file) {
|
||||
return new Parser(new LegacyLexer(version, source, file)).parse();
|
||||
}
|
||||
|
||||
public static Script parseScript(MuScriptVersion version, String source) {
|
||||
return parseScript(version, source, null);
|
||||
}
|
||||
|
||||
public static Script parseScript(MuScriptVersion version, String source, String file) {
|
||||
return new Parser(new LegacyLexer(version, source, file)).parseScript();
|
||||
}
|
||||
|
||||
public static Script parseMultiScript(MuScriptVersion version, String startFile, SourceFS filesystem) {
|
||||
return new Script(parseMultiScript(version, startFile, filesystem, new HashSet<>()).stream().flatMap(Script::stream).toList());
|
||||
}
|
||||
|
||||
private static List<Script> parseMultiScript(MuScriptVersion version, String startFile, SourceFS filesystem, Set<String> alreadyIncluded) {
|
||||
alreadyIncluded.add(startFile);
|
||||
boolean isIncludes = true;
|
||||
StringBuilder src = new StringBuilder();
|
||||
List<Script> includes = new LinkedList<>();
|
||||
int row = 0;
|
||||
final String includePrefix = "#include ";
|
||||
for (String s : filesystem.read(startFile).split("\n")) {
|
||||
row++;
|
||||
if (s.isBlank()) {
|
||||
src.append("\n");
|
||||
} else if (s.startsWith(includePrefix)) {
|
||||
if (isIncludes) {
|
||||
String file = s.substring(includePrefix.length());
|
||||
src.append("// include ").append(file).append("\n");
|
||||
if (!alreadyIncluded.contains(file)) {
|
||||
includes.addAll(parseMultiScript(version, file, filesystem, alreadyIncluded));
|
||||
}
|
||||
} else {
|
||||
throw new ParseException(PrettyPrintError.builder()
|
||||
.setLocation(new PrettyPrintError.Location(s, 0, row), new PrettyPrintError.Location(s, s.length() - 1, row))
|
||||
.setMessage("Includes MUST be located at the top of the file")
|
||||
.build());
|
||||
}
|
||||
} else {
|
||||
isIncludes = false;
|
||||
src.append(s).append("\n");
|
||||
}
|
||||
}
|
||||
includes.add(parseScript(version, src.toString(), startFile));
|
||||
return includes;
|
||||
}
|
||||
|
||||
public Parser(LexerImpl lexer) {
|
||||
this(new LegacyLexer(lexer));
|
||||
}
|
||||
|
||||
public Parser(Lexer lexer) {
|
||||
super(lexer.version());
|
||||
this.lexer = lexer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a single expression.
|
||||
* Must exhaust the source!
|
||||
*
|
||||
* @return the resulting expression
|
||||
*/
|
||||
public Expr parse() {
|
||||
advance();
|
||||
Expr expr = expression();
|
||||
if (!isAtEnd() && version.contains(MuScriptVersion.V2))
|
||||
throw new ParseException(PrettyPrintError.builder(lexer.location()).setMessage("Unexpected element after end of expression").build());
|
||||
return expr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a script instance.
|
||||
* Multiple instructions will be executed in sequence and the result of the last one will be returned.
|
||||
*
|
||||
* @return the resulting expression
|
||||
*/
|
||||
public Script parseScript() {
|
||||
advance();
|
||||
List<Expr> expressions = new LinkedList<>();
|
||||
while (!isAtEnd()) {
|
||||
expressions.add(expression());
|
||||
// Consume semicolon if present
|
||||
if (!lexer.wasNewlinePassed() & !match(Token.Semicolon) & !isAtEnd() & version.contains(MuScriptVersion.V3)) {
|
||||
throw error("Either a semicolon or a new line must separate expressions in scripts");
|
||||
}
|
||||
}
|
||||
if (expressions.isEmpty()) throw new ParseException(PrettyPrintError.builder(lexer.location()).setMessage("Missing any elements in closure").build());
|
||||
return new Script(expressions);
|
||||
}
|
||||
|
||||
// Expressions
|
||||
private Expr expression() {
|
||||
try {
|
||||
return conditional();
|
||||
} catch (RuntimeException e) {
|
||||
if (e instanceof ParseException) throw e;
|
||||
else throw error(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Expr conditional() {
|
||||
Expr expr = and();
|
||||
|
||||
if (match(Token.QuestionMark)) {
|
||||
CodeLocation location = previous.location();
|
||||
Expr trueExpr = expression();
|
||||
consume(Token.Colon, "Expected ':' after first part of condition");
|
||||
Expr falseExpr = expression();
|
||||
expr = new DynamicConditional(location, asBool(expr), ExprUtils.asDynamic(trueExpr), ExprUtils.asDynamic(falseExpr));
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr and() {
|
||||
Expr expr = or();
|
||||
|
||||
while (match(Token.And)) {
|
||||
CodeLocation location = previous.location();
|
||||
Expr right = or();
|
||||
expr = new And(location, asBool(expr), asBool(right));
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr or() {
|
||||
Expr expr = equality();
|
||||
|
||||
while (match(Token.Or)) {
|
||||
CodeLocation location = previous.location();
|
||||
Expr right = equality();
|
||||
expr = new Or(location, asBool(expr), asBool(right));
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr equality() {
|
||||
Expr expr = concat();
|
||||
|
||||
while (match(Token.EqualEqual, Token.BangEqual)) {
|
||||
Token op = previous.token();
|
||||
CodeLocation location = previous.location();
|
||||
Expr right = concat();
|
||||
BoolExpr e = new Equals(location, expr, right);
|
||||
if (op == Token.BangEqual) e = new Not(location, e);
|
||||
expr = e;
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr concat() {
|
||||
Expr expr = comparison();
|
||||
|
||||
while (match(Token.Concat)) {
|
||||
CodeLocation location = previous.location();
|
||||
Expr right = comparison();
|
||||
expr = new Concatenate(location, asString(expr), asString(right));
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr comparison() {
|
||||
Expr expr = term();
|
||||
|
||||
while (match(Token.Greater, Token.GreaterEqual, Token.Less, Token.LessEqual)) {
|
||||
Token op = previous.token();
|
||||
CodeLocation location = previous.location();
|
||||
NumberExpr right = asNumber(term());
|
||||
expr = switch (op) {
|
||||
case Greater -> new GreaterThan(location, asNumber(expr), right);
|
||||
case GreaterEqual -> new Not(location, new GreaterThan(location, right, asNumber(expr)));
|
||||
case Less -> new GreaterThan(location, right, asNumber(expr));
|
||||
case LessEqual -> new Not(location, new GreaterThan(location, asNumber(expr), right));
|
||||
default -> throw new IllegalStateException();
|
||||
};
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr term() {
|
||||
Expr expr = factor();
|
||||
|
||||
while (match(Token.Plus, Token.Minus)) {
|
||||
Token op = previous.token();
|
||||
CodeLocation location = previous.location();
|
||||
NumberExpr right = asNumber(factor());
|
||||
expr = switch (op) {
|
||||
case Plus -> new Add(location, asNumber(expr), right);
|
||||
case Minus -> new Subtract(location, asNumber(expr), right);
|
||||
default -> throw new IllegalStateException();
|
||||
};
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr factor() {
|
||||
Expr expr = exp();
|
||||
|
||||
while (match(Token.Star, Token.Slash, Token.Percentage)) {
|
||||
Token op = previous.token();
|
||||
CodeLocation location = previous.location();
|
||||
NumberExpr right = asNumber(exp());
|
||||
expr = switch (op) {
|
||||
case Star -> new Multiply(location, asNumber(expr), right);
|
||||
case Slash -> new Divide(location, asNumber(expr), right);
|
||||
case Percentage -> new Modulo(location, asNumber(expr), right);
|
||||
default -> throw new IllegalStateException();
|
||||
};
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr exp() {
|
||||
Expr expr = unary();
|
||||
|
||||
while (match(Token.UpArrow)) {
|
||||
CodeLocation location = previous.location();
|
||||
NumberExpr right = asNumber(unary());
|
||||
expr = new Power(location, asNumber(expr), right);
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr unary() {
|
||||
if (match(Token.Bang, Token.Minus)) {
|
||||
Token op = previous.token();
|
||||
CodeLocation location = previous.location();
|
||||
Expr right = unary();
|
||||
return switch (op) {
|
||||
case Bang -> new Not(location, asBool(right));
|
||||
case Minus -> new Negate(location, asNumber(right));
|
||||
default -> throw new IllegalStateException();
|
||||
};
|
||||
}
|
||||
|
||||
return call();
|
||||
}
|
||||
|
||||
private Expr call() {
|
||||
Expr expr = primary();
|
||||
|
||||
while (match(Token.LeftParen, Token.Dot, Token.LeftBracket, Token.DoubleColon)) {
|
||||
CodeLocation location = previous.location();
|
||||
expr = switch (previous.token()) {
|
||||
case LeftParen -> finishCall(location, expr);
|
||||
case Dot -> {
|
||||
Lexer.Token name = consume(Token.Identifier, "Expected field name after '.'");
|
||||
yield new Get(location, asDynamic(expr), Expr.literal(name.location(), name.lexeme()));
|
||||
}
|
||||
case DoubleColon -> {
|
||||
DynamicExpr callable;
|
||||
if (match(Token.Identifier)) {
|
||||
callable = new Variable(previous.location(), previous.lexeme());
|
||||
} else if (match(Token.LeftParen)) {
|
||||
Expr expr1 = expression();
|
||||
callable = ExprUtils.asDynamic(expr1);
|
||||
consume(Token.RightParen, "Expected ')' after expression");
|
||||
} else throw error("Bind operator requires right side to be a literal identifier or to be wrapped in parentheses.");
|
||||
yield new Bind(location, callable, ExprUtils.asDynamic(expr));
|
||||
}
|
||||
case LeftBracket -> {
|
||||
expr = new Get(location, asDynamic(expr), expression());
|
||||
consume(Token.RightBracket, "Expected closing bracket");
|
||||
yield expr;
|
||||
}
|
||||
default -> throw new IllegalStateException();
|
||||
};
|
||||
}
|
||||
|
||||
return expr;
|
||||
}
|
||||
|
||||
private Expr finishCall(CodeLocation location, Expr callee) {
|
||||
List<Call.Argument> args = new ArrayList<>(2);
|
||||
|
||||
if (!check(Token.RightParen)) {
|
||||
do {
|
||||
args.add(new Call.Argument(asDynamic(expression()), match(Token.Ellipsis)));
|
||||
} while (match(Token.Comma));
|
||||
}
|
||||
|
||||
consume(Token.RightParen, "Expected ')' after function arguments");
|
||||
return new Call(location, asDynamic(callee), args);
|
||||
}
|
||||
|
||||
private Expr primary() {
|
||||
if (match(Token.Null)) return Expr.literalNull(previous.location());
|
||||
if (match(Token.String)) return "this".equals(previous.lexeme()) ? new This(previous.location()) : Expr.literal(previous.location(), previous.lexeme());
|
||||
if (match(Token.True, Token.False)) return Expr.literal(previous.location(), previous.lexeme().equals("true"));
|
||||
if (match(Token.Number)) return Expr.literal(previous.location(), Double.parseDouble(previous.lexeme()));
|
||||
if (match(Token.Identifier)) {
|
||||
CodeLocation location = previous.location();
|
||||
String name = previous.lexeme();
|
||||
if (match(Token.Assign)) {
|
||||
Expr expr = expression();
|
||||
return new DynamicAssign(location, name, ExprUtils.asDynamic(expr));
|
||||
}
|
||||
else return new Variable(location, name);
|
||||
}
|
||||
|
||||
if (match(Token.LeftParen)) {
|
||||
Expr expr = expression();
|
||||
consume(Token.RightParen, "Expected ')' after expression");
|
||||
return expr;
|
||||
}
|
||||
|
||||
if (match(Token.LeftBrace)) {
|
||||
int start = previous.start();
|
||||
if (match(Token.Arrow)) return finishClosure(start, null, false);
|
||||
if (match(Token.RightBrace)) return new ObjectLiteral(location(start, previous.start()), Map.of());
|
||||
consume(Token.Identifier, "Expected arrow or identifier as first element in closure or object");
|
||||
String first = previous.lexeme();
|
||||
if (check(Token.Arrow)) return finishClosure(start, first, false);
|
||||
if (match(Token.Ellipsis)) return finishClosure(start, first, true);
|
||||
if (check(Token.Comma)) return finishClosure(start, first, false);
|
||||
if (match(Token.Assign)) {
|
||||
Expr expr = expression();
|
||||
return finishObject(start, first, ExprUtils.asDynamic(expr));
|
||||
}
|
||||
throw error("Unexpected");
|
||||
}
|
||||
|
||||
throw error("Expected expression.");
|
||||
}
|
||||
|
||||
private Expr finishClosure(int start, @Nullable String firstArg, boolean firstVariadic) {
|
||||
List<String> boundArgs = new LinkedList<>();
|
||||
boolean variadic = false;
|
||||
if (firstArg != null) {
|
||||
boundArgs.add(firstArg);
|
||||
if (firstVariadic) {
|
||||
consume(Token.Arrow, "Variadic argument MUST be the last argument");
|
||||
variadic = true;
|
||||
} else {
|
||||
while (!match(Token.Arrow)) {
|
||||
consume(Token.Comma, "Closure parameters MUST be comma-seperated");
|
||||
consume(Token.Identifier, "Closure arguments MUST be identifiers");
|
||||
boundArgs.add(previous.lexeme());
|
||||
if (match(Token.Ellipsis)) {
|
||||
variadic = true;
|
||||
consume(Token.Arrow, "Variadic argument MUST be the last argument");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
List<Expr> expressions = new LinkedList<>();
|
||||
while (!match(Token.RightBrace)) {
|
||||
expressions.add(expression());
|
||||
// Consume semicolon if present
|
||||
if (!lexer.wasNewlinePassed() & !match(Token.Semicolon) & version.contains(MuScriptVersion.V3)) {
|
||||
if (match(Token.RightBrace)) break;
|
||||
throw error("Either a semicolon or a new line must separate expressions in closures");
|
||||
}
|
||||
}
|
||||
int end = previous.start();
|
||||
return new Closure(location(start, end), boundArgs, variadic, expressions);
|
||||
}
|
||||
|
||||
private Expr finishObject(int start, @Nullable String firstArg, @Nullable DynamicExpr firstValue) {
|
||||
Map<String, DynamicExpr> content = new LinkedHashMap<>();
|
||||
content.put(firstArg, firstValue);
|
||||
while (match(Token.Comma)) {
|
||||
consume(Token.Identifier, "Object element MUST start with an identifier");
|
||||
String name = previous.lexeme();
|
||||
consume(Token.Assign, "Object element name and value MUST be seperated with '='");
|
||||
Expr expr = expression();
|
||||
content.put(name, ExprUtils.asDynamic(expr));
|
||||
}
|
||||
consume(Token.RightBrace, "Expected end of object");
|
||||
return new ObjectLiteral(location(start, previous.start()), content);
|
||||
}
|
||||
|
||||
// Type conversion
|
||||
private BoolExpr asBool(Expr expression) {
|
||||
try {
|
||||
return ExprUtils.asBool(expression);
|
||||
} catch (TypeMismatchException e) {
|
||||
throw error(e.getMessage(), expression);
|
||||
}
|
||||
}
|
||||
|
||||
private NumberExpr asNumber(Expr expression) {
|
||||
try {
|
||||
return ExprUtils.asNumber(expression);
|
||||
} catch (TypeMismatchException e) {
|
||||
throw error(e.getMessage(), expression);
|
||||
}
|
||||
}
|
||||
|
||||
private StringExpr asString(Expr expression) {
|
||||
try {
|
||||
return ExprUtils.asString(expression);
|
||||
} catch (TypeMismatchException e) {
|
||||
throw error(e.getMessage(), expression);
|
||||
}
|
||||
}
|
||||
|
||||
private DynamicExpr asDynamic(Expr expression) {
|
||||
return ExprUtils.asDynamic(expression);
|
||||
}
|
||||
|
||||
// Helpers
|
||||
private CodeLocation location(int chStart, int chEnd) {
|
||||
return new CodeLocation(chStart, chEnd, lexer.getSource(), lexer.getFile());
|
||||
}
|
||||
|
||||
private ParseException error(String message) {
|
||||
int loc = lexer.getPrevious().current() - 1;
|
||||
return new ParseException(PrettyPrintError.builder(location(loc, loc)).setMessage(message).build());
|
||||
}
|
||||
|
||||
private ParseException error(String message, Expr expr) {
|
||||
return new ParseException(PrettyPrintError.builder(expr.location()).setMessage(message).build());
|
||||
}
|
||||
|
||||
private Lexer.Token consume(Token token, String message) {
|
||||
if (check(token)) return advance();
|
||||
throw error(message + " but got " + lexer.getPrevious().token());
|
||||
}
|
||||
|
||||
private boolean match(Token... tokens) {
|
||||
for (Token token : tokens) {
|
||||
if (check(token)) {
|
||||
advance();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean check(Token token) {
|
||||
if (isAtEnd()) return false;
|
||||
return lexer.getPrevious().token() == token;
|
||||
}
|
||||
|
||||
private Lexer.Token advance() {
|
||||
previous = lexer.getPrevious();
|
||||
|
||||
lexer.advance();
|
||||
|
||||
if (lexer.getPrevious().token() == Token.Error) {
|
||||
throw error(lexer.getPrevious().lexeme());
|
||||
}
|
||||
|
||||
return previous;
|
||||
}
|
||||
|
||||
private boolean isAtEnd() {
|
||||
return lexer.getPrevious().token() == Token.EOF;
|
||||
}
|
||||
|
||||
// Parse Exception
|
||||
public static class ParseException extends RuntimeException {
|
||||
public final PrettyPrintError error;
|
||||
|
||||
public ParseException(PrettyPrintError error) {
|
||||
super(error.toString());
|
||||
this.error = error;
|
||||
}
|
||||
|
||||
public ParseException(PrettyPrintError error, Throwable cause) {
|
||||
super(error.toString(), cause);
|
||||
this.error = error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package io.gitlab.jfronny.muscript.parser;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public interface SourceFS {
|
||||
String read(String file);
|
||||
|
||||
record Immutable(Map<String, String> files) implements SourceFS {
|
||||
@Override
|
||||
public String read(String file) {
|
||||
return files.get(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
package io.gitlab.jfronny.muscript.parser;
|
||||
|
||||
public enum Type {
|
||||
String, Number, Boolean, Dynamic
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package io.gitlab.jfronny.muscript.parser;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public abstract class VersionedComponent {
|
||||
protected final MuScriptVersion version;
|
||||
|
||||
public VersionedComponent(MuScriptVersion version) {
|
||||
this.version = Objects.requireNonNull(version);
|
||||
}
|
||||
|
||||
public MuScriptVersion getComponentVersion() {
|
||||
return version;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package io.gitlab.jfronny.muscript.parser.impl;
|
||||
|
||||
import io.gitlab.jfronny.muscript.ast.Expr;
|
||||
import io.gitlab.jfronny.muscript.ast.context.IExprParser;
|
||||
import io.gitlab.jfronny.muscript.parser.MuScriptVersion;
|
||||
import io.gitlab.jfronny.muscript.parser.Parser;
|
||||
|
||||
public class ExprParserImpl implements IExprParser {
|
||||
@Override
|
||||
public Expr parse(String expr) {
|
||||
return Parser.parse(MuScriptVersion.DEFAULT, expr);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
package io.gitlab.jfronny.muscript.parser.lexer;
|
||||
|
||||
import io.gitlab.jfronny.muscript.core.CodeLocation;
|
||||
import io.gitlab.jfronny.muscript.parser.MuScriptVersion;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
/**
|
||||
* Wraps the old Lexer implementation in the new Lexer interface for compatibility
|
||||
*/
|
||||
public class LegacyLexer implements Lexer {
|
||||
private final LexerImpl backend;
|
||||
private Token previous = null;
|
||||
|
||||
public LegacyLexer(MuScriptVersion version, String source) {
|
||||
this(new LexerImpl(version, source));
|
||||
}
|
||||
|
||||
public LegacyLexer(MuScriptVersion version, String source, String file) {
|
||||
this(new LexerImpl(version, source, file));
|
||||
}
|
||||
|
||||
public LegacyLexer(LexerImpl backend) {
|
||||
this.backend = backend;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CodeLocation location() {
|
||||
return backend.location();
|
||||
}
|
||||
|
||||
@Override
|
||||
public MuScriptVersion version() {
|
||||
return backend.getComponentVersion();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean wasNewlinePassed() {
|
||||
return backend.passedNewline;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Token getPrevious() {
|
||||
return previous;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Token advance() {
|
||||
backend.next();
|
||||
return previous = new Token(
|
||||
backend.lexeme,
|
||||
backend.token,
|
||||
backend.start,
|
||||
backend.current,
|
||||
backend.ch,
|
||||
new CodeLocation(backend.start, backend.current - 1, backend.source, backend.file)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public @Nullable String getSource() {
|
||||
return backend.source;
|
||||
}
|
||||
|
||||
@Override
|
||||
public @Nullable String getFile() {
|
||||
return backend.file;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package io.gitlab.jfronny.muscript.parser.lexer;
|
||||
|
||||
import io.gitlab.jfronny.muscript.core.CodeLocation;
|
||||
import io.gitlab.jfronny.muscript.parser.MuScriptVersion;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public interface Lexer {
|
||||
CodeLocation location();
|
||||
MuScriptVersion version();
|
||||
boolean wasNewlinePassed();
|
||||
Token getPrevious();
|
||||
Token advance();
|
||||
|
||||
@Nullable String getSource();
|
||||
@Nullable String getFile();
|
||||
|
||||
record Token(String lexeme, io.gitlab.jfronny.muscript.parser.lexer.Token token, int start, int current, char ch, CodeLocation location) {
|
||||
@Override
|
||||
public String toString() {
|
||||
return token + " '" + lexeme + "'";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,268 @@
|
|||
package io.gitlab.jfronny.muscript.parser.lexer;
|
||||
|
||||
import io.gitlab.jfronny.muscript.core.CodeLocation;
|
||||
import io.gitlab.jfronny.muscript.parser.MuScriptVersion;
|
||||
import io.gitlab.jfronny.muscript.parser.VersionedComponent;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
//TODO refactor to use the new Lexer interface directly
|
||||
/**
|
||||
* The lexer for muScript, heavily inspired by starscript
|
||||
*/
|
||||
public class LexerImpl extends VersionedComponent {
|
||||
public final String file;
|
||||
|
||||
/**
|
||||
* The type of the token
|
||||
*/
|
||||
public Token token;
|
||||
/**
|
||||
* The string representation of the token
|
||||
*/
|
||||
public String lexeme;
|
||||
|
||||
public char ch;
|
||||
|
||||
public final String source;
|
||||
public int start, current;
|
||||
|
||||
public boolean passedNewline = false;
|
||||
|
||||
public LexerImpl(MuScriptVersion version, String source) {
|
||||
this(version, source, null);
|
||||
}
|
||||
|
||||
public LexerImpl(MuScriptVersion version, String source, String file) {
|
||||
super(version);
|
||||
this.source = Objects.requireNonNull(source);
|
||||
this.file = file;
|
||||
}
|
||||
|
||||
public CodeLocation location() {
|
||||
return new CodeLocation(start, current - 1, source, file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans for the next token storing it in {@link LexerImpl#token} and {@link LexerImpl#lexeme}. Produces {@link Token#EOF} if the end of source code has been reached and {@link Token#Error} if there has been an error
|
||||
*/
|
||||
public void next() {
|
||||
start = current;
|
||||
passedNewline = false;
|
||||
|
||||
if (isAtEnd()) {
|
||||
createToken(Token.EOF);
|
||||
return;
|
||||
}
|
||||
|
||||
// Scan expression
|
||||
skipWhitespaceAndComments();
|
||||
if (isAtEnd()) {
|
||||
createToken(Token.EOF);
|
||||
return;
|
||||
}
|
||||
|
||||
char c = advance();
|
||||
|
||||
if (isDigit(c)) number();
|
||||
else if (isIdentifier(c)) identifier(false);
|
||||
else {
|
||||
switch (c) {
|
||||
case '\'', '"' -> string(c);
|
||||
|
||||
case '`' -> identifier(true);
|
||||
|
||||
case '=' -> {
|
||||
if (match('=')) createToken(Token.EqualEqual);
|
||||
else createToken(Token.Assign);
|
||||
}
|
||||
case '!' -> createToken(match('=') ? Token.BangEqual : Token.Bang);
|
||||
|
||||
case '+' -> createToken(Token.Plus);
|
||||
case '-' -> {
|
||||
if (match('>')) createToken(Token.Arrow);
|
||||
else createToken(Token.Minus);
|
||||
}
|
||||
case '*' -> createToken(Token.Star);
|
||||
case '/' -> createToken(Token.Slash);
|
||||
case '%' -> createToken(Token.Percentage);
|
||||
case '>' -> createToken(match('=') ? Token.GreaterEqual : Token.Greater);
|
||||
case '<' -> createToken(match('=') ? Token.LessEqual : Token.Less);
|
||||
|
||||
case '&' -> createToken(Token.And);
|
||||
case '|' -> createToken(match('|') ? Token.Concat : Token.Or);
|
||||
case '^' -> createToken(Token.UpArrow);
|
||||
|
||||
case '.' -> {
|
||||
if (match('.')) {
|
||||
if (match('.')) createToken(Token.Ellipsis);
|
||||
else createToken(Token.Error, "Unexpected '..', did you mean '...'?");
|
||||
} else createToken(Token.Dot);
|
||||
}
|
||||
case ',' -> createToken(Token.Comma);
|
||||
case '?' -> createToken(Token.QuestionMark);
|
||||
case ':' -> createToken(match(':') ? Token.DoubleColon : Token.Colon);
|
||||
case '(' -> createToken(Token.LeftParen);
|
||||
case ')' -> createToken(Token.RightParen);
|
||||
case '[' -> createToken(Token.LeftBracket);
|
||||
case ']' -> createToken(Token.RightBracket);
|
||||
case '{' -> createToken(Token.LeftBrace);
|
||||
case '}' -> createToken(Token.RightBrace);
|
||||
|
||||
case ';' -> {
|
||||
createToken(Token.Semicolon);
|
||||
passedNewline = true;
|
||||
}
|
||||
|
||||
default -> unexpected();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void string(char stringChar) {
|
||||
while (!isAtEnd() && peek() != stringChar) {
|
||||
advance();
|
||||
}
|
||||
|
||||
if (isAtEnd()) {
|
||||
createToken(Token.Error, "Unterminated expression");
|
||||
} else {
|
||||
advance();
|
||||
createToken(Token.String, source.substring(start + 1, current - 1));
|
||||
}
|
||||
}
|
||||
|
||||
private void number() {
|
||||
while (isDigit(peek())) advance();
|
||||
|
||||
if (peek() == '.' && isDigit(peekNext())) {
|
||||
advance();
|
||||
|
||||
while (isDigit(peek())) advance();
|
||||
}
|
||||
|
||||
createToken(Token.Number);
|
||||
}
|
||||
|
||||
private void identifier(boolean explicit) {
|
||||
if (explicit) {
|
||||
while (!isAtEnd() && peek() != '`') {
|
||||
advance();
|
||||
}
|
||||
if (isAtEnd()) {
|
||||
createToken(Token.Error, "Unterminated expression");
|
||||
} else {
|
||||
advance();
|
||||
createToken(Token.Identifier, source.substring(start + 1, current - 1));
|
||||
}
|
||||
} else {
|
||||
while (!isAtEnd()) {
|
||||
char c = peek();
|
||||
if (isIdentifier(c) || isDigit(c)) {
|
||||
advance();
|
||||
} else break;
|
||||
}
|
||||
|
||||
createToken(Token.Identifier);
|
||||
|
||||
switch (lexeme) {
|
||||
case "null" -> token = Token.Null;
|
||||
case "true" -> token = Token.True;
|
||||
case "false" -> token = Token.False;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void skipWhitespaceAndComments() {
|
||||
while (true) {
|
||||
if (isAtEnd()) return;
|
||||
|
||||
switch (peek()) {
|
||||
case ' ', '\t' -> advance();
|
||||
case '\r', '\n' -> {
|
||||
advance();
|
||||
passedNewline = true;
|
||||
}
|
||||
case '/' -> {
|
||||
switch (peekNext()) {
|
||||
case '/' -> {
|
||||
while (!isAtEnd() && peek() != '\r' && peek() != '\n') advance();
|
||||
}
|
||||
case '*' -> {
|
||||
advance();
|
||||
advance();
|
||||
while (!isAtEnd() && (peek() != '*' || peekNext() != '/')) advance();
|
||||
if (!isAtEnd()) {
|
||||
advance();
|
||||
advance();
|
||||
}
|
||||
}
|
||||
default -> {
|
||||
start = current;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
default -> {
|
||||
start = current;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helpers
|
||||
|
||||
private void unexpected() {
|
||||
createToken(Token.Error, "Unexpected character");
|
||||
}
|
||||
|
||||
private void createToken(Token token, String lexeme) {
|
||||
this.token = token;
|
||||
this.lexeme = lexeme;
|
||||
}
|
||||
|
||||
private void createToken(Token token) {
|
||||
createToken(token, source.substring(start, current));
|
||||
}
|
||||
|
||||
private boolean match(char expected) {
|
||||
if (isAtEnd()) return false;
|
||||
if (source.charAt(current) != expected) return false;
|
||||
|
||||
advance();
|
||||
return true;
|
||||
}
|
||||
|
||||
private char advance() {
|
||||
return ch = source.charAt(current++);
|
||||
}
|
||||
|
||||
private char peek() {
|
||||
if (isAtEnd()) return '\0';
|
||||
return source.charAt(current);
|
||||
}
|
||||
|
||||
private char peekNext() {
|
||||
if (current + 1 >= source.length()) return '\0';
|
||||
return source.charAt(current + 1);
|
||||
}
|
||||
|
||||
private boolean isAtEnd() {
|
||||
return current >= source.length();
|
||||
}
|
||||
|
||||
private boolean isDigit(char c) {
|
||||
return c >= '0' && c <= '9';
|
||||
}
|
||||
|
||||
private boolean isIdentifier(char c) {
|
||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '$';
|
||||
}
|
||||
|
||||
// Visualization
|
||||
@Override
|
||||
public String toString() {
|
||||
return source.substring(0, start) + '[' + source.substring(start, current) + ']' + source.substring(current);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package io.gitlab.jfronny.muscript.parser.lexer;
|
||||
|
||||
public enum Token {
|
||||
String, Identifier, Number,
|
||||
|
||||
Null,
|
||||
True, False,
|
||||
And, Or,
|
||||
|
||||
Assign,
|
||||
EqualEqual, BangEqual,
|
||||
|
||||
Concat,
|
||||
|
||||
Greater, GreaterEqual,
|
||||
Less, LessEqual,
|
||||
|
||||
Plus, Minus,
|
||||
Star, Slash, Percentage, UpArrow,
|
||||
Bang,
|
||||
|
||||
Dot, Ellipsis, Comma,
|
||||
QuestionMark, Colon,
|
||||
LeftParen, RightParen,
|
||||
LeftBracket, RightBracket,
|
||||
|
||||
LeftBrace, RightBrace,
|
||||
Arrow,
|
||||
|
||||
Semicolon,
|
||||
|
||||
DoubleColon,
|
||||
|
||||
Error, EOF
|
||||
}
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
package io.gitlab.jfronny.muscript.parser.util;
|
||||
|
||||
import io.gitlab.jfronny.muscript.core.CodeLocation;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Class for storing errors with code context
|
||||
* Can be generated from a LocationalException with asPrintable
|
||||
*
|
||||
* @param message The error message
|
||||
* @param start The location of the start of this error. May be null.
|
||||
* @param end The location of the end of this error. May be null.
|
||||
*/
|
||||
public record PrettyPrintError(String message, @Nullable Location start, @Nullable Location end, List<? extends StackFrame> callStack) {
|
||||
/**
|
||||
* A location in the source code
|
||||
*
|
||||
* @param line The code of the line this is in
|
||||
* @param column The column of this character, starting with 0
|
||||
* @param row The row of this line, starting with 1
|
||||
*/
|
||||
public record Location(String line, int column, int row) {
|
||||
public static Location create(@Nullable String source, int index) {
|
||||
if (source == null || index < 0) return new Location("", -1, -1);
|
||||
int lineStart = source.lastIndexOf('\n', index);
|
||||
int lineEnd = source.indexOf('\n', index);
|
||||
if (lineEnd == -1) lineEnd = source.length();
|
||||
|
||||
int lineIndex = lineStart > 0 ? (int) source.substring(0, lineStart).chars().filter(c -> c == '\n').count() : 0;
|
||||
int column = index - lineStart - 1;
|
||||
|
||||
String line = lineStart == source.length() - 1 ? "" : source.substring(lineStart + 1, lineEnd);
|
||||
|
||||
return new Location(line, column, lineIndex + 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
if (column >= line.length()) return "character " + (column + 1) + " of line " + row;
|
||||
else return "'" + line.charAt(column) + "' (character " + (column + 1) + ")";
|
||||
}
|
||||
|
||||
public String prettyPrint() {
|
||||
String linePrefix = String.format("%1$6d", row) + " | ";
|
||||
return linePrefix + line + "\n" + " ".repeat(linePrefix.length() + column) + "^-- ";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts this error to a human-readable error message
|
||||
*
|
||||
* @return A string representation
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
if (start == null || start.column < 0) sb.append("Error at unknown location: ").append(message);
|
||||
else {
|
||||
sb.append("Error at ").append(start).append(": ").append(message).append("\n");
|
||||
if (end == null) sb.append(start.prettyPrint()).append("Here");
|
||||
else if (start.row == end.row) {
|
||||
String linePrefix = String.format("%1$6d", start.row) + " | ";
|
||||
sb.append(linePrefix).append(start.line).append("\n").append(" ".repeat(linePrefix.length() + start.column)).append("^").append("-".repeat(end.column - start.column - 1)).append("^-- Here");
|
||||
}
|
||||
else sb.append(start.prettyPrint()).append("From here").append(end.prettyPrint()).append("to here");
|
||||
}
|
||||
callStack.forEach(frame -> sb.append("\n at ").append(frame));
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static Builder builder(CodeLocation location) {
|
||||
return builder().setSource(location.source()).setLocation(location.chStart(), location.chEnd());
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String message;
|
||||
private Location start;
|
||||
private @Nullable Location end;
|
||||
private List<? extends StackFrame> callStack = List.of();
|
||||
private String source;
|
||||
|
||||
public Builder setSource(String source) {
|
||||
this.source = source;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setMessage(String message) {
|
||||
this.message = message;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setLocation(int chStart) {
|
||||
return setLocation(Location.create(source, chStart), null);
|
||||
}
|
||||
|
||||
public Builder setLocation(int chStart, int chEnd) {
|
||||
if (chEnd == chStart) return setLocation(chStart);
|
||||
if (chEnd < chStart) {
|
||||
int a = chEnd;
|
||||
chEnd = chStart;
|
||||
chStart = a;
|
||||
}
|
||||
return setLocation(Location.create(source, chStart), Location.create(source, chEnd));
|
||||
}
|
||||
|
||||
public Builder setLocation(Location start, Location end) {
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setCallStack(List<StackFrame> callStack) {
|
||||
this.callStack = callStack.stream()
|
||||
.map(frame -> frame instanceof StackFrame.Lined lined
|
||||
? lined
|
||||
: source == null ? frame : ((StackFrame.Raw) frame).lined(source))
|
||||
.toList();
|
||||
return this;
|
||||
}
|
||||
|
||||
public PrettyPrintError build() {
|
||||
return new PrettyPrintError(message, start, end, callStack);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package io.gitlab.jfronny.muscript.parser.util;
|
||||
|
||||
public sealed interface StackFrame {
|
||||
record Raw(String file, String name, int chStart) implements StackFrame {
|
||||
@Override
|
||||
public String toString() {
|
||||
return name + " (call: character " + chStart + (file == null ? ")" : " in " + file + ")");
|
||||
}
|
||||
|
||||
public Lined lined(String source) {
|
||||
int lineStart = source.lastIndexOf('\n', chStart);
|
||||
int lineIndex = lineStart > 0 ? (int) source.substring(0, lineStart).chars().filter(c -> c == '\n').count() : 0;
|
||||
return new Lined(file, name, lineIndex + 1);
|
||||
}
|
||||
}
|
||||
|
||||
record Lined(String file, String name, int row) implements StackFrame {
|
||||
@Override
|
||||
public String toString() {
|
||||
return name + " (call: line " + row + (file == null ? ")" : " in " + file + ")");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
module io.gitlab.jfronny.commons.muscript.parser {
|
||||
requires io.gitlab.jfronny.commons;
|
||||
requires static org.jetbrains.annotations;
|
||||
requires io.gitlab.jfronny.commons.muscript.ast;
|
||||
requires io.gitlab.jfronny.commons.muscript.core;
|
||||
exports io.gitlab.jfronny.muscript.parser;
|
||||
exports io.gitlab.jfronny.muscript.parser.lexer;
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
io.gitlab.jfronny.muscript.parser.impl.ExprParserImpl
|
||||
Loading…
Reference in New Issue