Changed how we deal w/ StringLiterals across multiple lines

master
bill 4 years ago
parent 997178ba9c
commit 03c393beff

@ -145,8 +145,8 @@ action code {:
terminal NEWLINE; terminal NEWLINE;
terminal String PLUS; terminal String PLUS;
terminal String MINUS; terminal String MINUS;
terminal String MULTIPLY; terminal String MUL;
terminal String DIVIDE; terminal String DIV;
terminal String NAMES; terminal String NAMES;
terminal Integer NUMBER; terminal Integer NUMBER;

@ -8,7 +8,7 @@ import java.util.ArrayList;
%unicode %unicode
%line %line
%column %column
%states AFTER %states AFTER, STR
%class ChocoPyLexer %class ChocoPyLexer
%public %public
@ -33,6 +33,7 @@ import java.util.ArrayList;
/** Producer of token-related values for the parser. */ /** Producer of token-related values for the parser. */
final ComplexSymbolFactory symbolFactory = new ComplexSymbolFactory(); final ComplexSymbolFactory symbolFactory = new ComplexSymbolFactory();
private int currIndent = 0; private int currIndent = 0;
private int currString = "";
private ArrayList<Integer> stack = new ArrayList<Integer>(20); private ArrayList<Integer> stack = new ArrayList<Integer>(20);
/** Return a terminal symbol of syntactic category TYPE and no /** Return a terminal symbol of syntactic category TYPE and no
* semantic value at the current source location. */ * semantic value at the current source location. */
@ -67,7 +68,7 @@ WhiteSpace = [ \t]
LineBreak = \r|\n|\r\n LineBreak = \r|\n|\r\n
IntegerLiteral = 0|[1-9][0-9]* IntegerLiteral = 0|[1-9][0-9]*
StringLiteral = \"([^\"\\]|(\\\")|(\\t)|(\\r)|(\\n)|(\\\\))*\" StringLiteral = ([^\"\\]|(\\\")|(\\t)|(\\r)|(\\n)|(\\\\))*
Names = (_|[a-z]|[A-Z])(_|[a-z]|[A-Z])* Names = (_|[a-z]|[A-Z])(_|[a-z]|[A-Z])*
Comments = #[^\r\n]* Comments = #[^\r\n]*
%% %%
@ -112,7 +113,8 @@ Comments = #[^\r\n]*
/* Literals. */ /* Literals. */
{IntegerLiteral} { return symbol(ChocoPyTokens.NUMBER, {IntegerLiteral} { return symbol(ChocoPyTokens.NUMBER,
Integer.parseInt(yytext())); } Integer.parseInt(yytext())); }
{StringLiteral} { return symbol(ChocoPyTokens.STRING, yytext());} // {StringLiteral} { return symbol(ChocoPyTokens.STRING, yytext());}
"\"" {yystart(STR); currString = "";}
"False" { return symbol(ChocoPyTokens.BOOL, false); } "False" { return symbol(ChocoPyTokens.BOOL, false); }
"True" { return symbol(ChocoPyTokens.BOOL, true); } "True" { return symbol(ChocoPyTokens.BOOL, true); }
"None" { return symbol(ChocoPyTokens.NONE); } "None" { return symbol(ChocoPyTokens.NONE); }
@ -180,7 +182,10 @@ Comments = #[^\r\n]*
/* Comment. */ /* Comment. */
{Comments} { /* ignore */ } {Comments} { /* ignore */ }
} }
<STR>{
{StringLiteral} {currString+=yytext();}
"\"" {yybegin(AFTER); return symbol(ChocoPyTokens.STRING, currString);}
}
<<EOF>> { return symbol(ChocoPyTokens.EOF); } <<EOF>> { return symbol(ChocoPyTokens.EOF); }
/* Error fallback. */ /* Error fallback. */

Loading…
Cancel
Save