diff --git a/src/main/cup/chocopy/pa1/ChocoPy.cup b/src/main/cup/chocopy/pa1/ChocoPy.cup index 0585cf7..d4ff444 100644 --- a/src/main/cup/chocopy/pa1/ChocoPy.cup +++ b/src/main/cup/chocopy/pa1/ChocoPy.cup @@ -144,11 +144,6 @@ action code {: */ terminal NEWLINE; terminal String PLUS; -terminal String MINUS; -terminal String MUL; -terminal String DIV; -terminal String NAMES; - terminal Integer NUMBER; /* Returned by the lexer for erroneous tokens. Since it does not appear in * the grammar, it indicates a syntax error. */ diff --git a/src/main/jflex/chocopy/pa1/ChocoPy.jflex b/src/main/jflex/chocopy/pa1/ChocoPy.jflex index ebcec10..9aafe7f 100644 --- a/src/main/jflex/chocopy/pa1/ChocoPy.jflex +++ b/src/main/jflex/chocopy/pa1/ChocoPy.jflex @@ -1,6 +1,6 @@ package chocopy.pa1; import java_cup.runtime.*; -import java.util.ArrayList; + %% /*** Do not change the flags below unless you know what you are doing. ***/ @@ -8,7 +8,7 @@ import java.util.ArrayList; %unicode %line %column -%states AFTER, STR + %class ChocoPyLexer %public @@ -32,13 +32,11 @@ import java.util.ArrayList; /** Producer of token-related values for the parser. */ final ComplexSymbolFactory symbolFactory = new ComplexSymbolFactory(); - private int currIndent = 0; - private int currString = ""; - private ArrayList stack = new ArrayList(20); + /** Return a terminal symbol of syntactic category TYPE and no * semantic value at the current source location. */ private Symbol symbol(int type) { - return symbol(type); + return symbol(type, yytext()); } /** Return a terminal symbol of syntactic category TYPE and semantic @@ -49,17 +47,7 @@ import java.util.ArrayList; new ComplexSymbolFactory.Location(yyline + 1,yycolumn + yylength()), value); } - private void push(int indent){ - stack.add(indent); - } - private int pop(){ - if(stack.isEmpty()) return 0; - return stack.remove(stack.size() - 1); - } - private int top(){ - if(stack.isEmpty) return 0; - return stack.get(stack.size() - 1); - } + %} /* Macros (regexes used in rules below) */ @@ -67,125 +55,27 @@ import java.util.ArrayList; WhiteSpace = [ \t] LineBreak = \r|\n|\r\n -IntegerLiteral = 0|[1-9][0-9]* -StringLiteral = ([^\"\\]|(\\\")|(\\t)|(\\r)|(\\n)|(\\\\))* -Names = (_|[a-z]|[A-Z])(_|[a-z]|[A-Z])* -Comments = #[^\r\n]* +IntegerLiteral = 0 | [1-9][0-9]* + %% -{ - {WhiteSpace} - { - String space = yytext(); - if(space == "\t") - currIndent += 8; - else - currIndent ++; - } - {LineBreak} - { - currIndent = 0; - } - {Comments} { /* ignored */ } - [^ \t\r\n#] - { - yypushback(1); - if(top() > currIndent) - { - pop(); - return symbol(ChocoPyTokens.DEDENT); - } - yystart(AFTER); - if(top()< currIndent) - { - push(currIndent); - return symbol(ChocoPyTokens.INDENT); - } - } -} - { + + { /* Delimiters. */ - {LineBreak} { return symbol(ChocoPyTokens.NEWLINE); yybegin(YYINITIAL); currIndent = 0;} - ":" { return symbol(ChocoPyTokens.COLON); } - "," { return symbol(ChocoPyTokens.COMMA); } + {LineBreak} { return symbol(ChocoPyTokens.NEWLINE); } /* Literals. */ {IntegerLiteral} { return symbol(ChocoPyTokens.NUMBER, Integer.parseInt(yytext())); } -// {StringLiteral} { return symbol(ChocoPyTokens.STRING, yytext());} - "\"" {yystart(STR); currString = "";} - "False" { return symbol(ChocoPyTokens.BOOL, false); } - "True" { return symbol(ChocoPyTokens.BOOL, true); } - "None" { return symbol(ChocoPyTokens.NONE); } - - /*Keywords*/ - "if" {return symbol(ChocoPyTokens.IF);} - "else" {return symbol(ChocoPyTokens.ELSE);} - "elif" {return symbol(ChocoPyTokens.ELIF);} - "while" {return symbol(ChocoPyTokens.WHILE);} - "class" {return symbol(ChocoPyTokens.CLASS);} - "def" {return symbol(ChocoPyTokens.DEF);} - "lambda" {return symbol(ChocoPyTokens.LAMBDA);} - "as" { return symbol(ChocoPyTokens.AS); } - "for" { return symbol(ChocoPyTokens.FOR); } - "global" { return symbol(ChocoPyTokens.GLOBAL); } - "in" { return symbol(ChocoPyTokens.IN); } - "nonlocal" { return symbol(ChocoPyTokens.NONLOCAL); } - "pass" { return symbol(ChocoPyTokens.PASS); } - "return" { return symbol(ChocoPyTokens.RETURN); } - "assert" { return symbol(ChocoPyTokens.ASSERT); } - "await" { return symbol(ChocoPyTokens.AWAIT); } - "break" { return symbol(ChocoPyTokens.BREAK); } - "continue" { return symbol(ChocoPyTokens.CONTINUE); } - "del" { return symbol(ChocoPyTokens.DEL); } - "except" { return symbol(ChocoPyTokens.EXCEPT); } - "finally" { return symbol(ChocoPyTokens.FINALLY); } - "from" { return symbol(ChocoPyTokens.FROM); } - "import" { return symbol(ChocoPyTokens.IMPORT); } - "raise" { return symbol(ChocoPyTokens.RAISE); } - "try" { return symbol(ChocoPyTokens.TRY); } - "with" { return symbol(ChocoPyTokens.WITH); } - "yield" { return symbol(ChocoPyTokens.YIELD); } - /* Operators. */ - "+" { return symbol(ChocoPyTokens.PLUS); } - "-" { return symbol(ChocoPyTokens.MINUS); } - "*" { return symbol(ChocoPyTokens.MUL); } - "//" { return symbol(ChocoPyTokens.DIV); } - "/" { return symbol(ChocoPyTokens.DIV); } - "%" { return symbol(ChocoPyTokens.MOD); } - ">" { return symbol(ChocoPyTokens.GT); } - "<" { return symbol(ChocoPyTokens.LT); } - "==" { return symbol(ChocoPyTokens.EQUAL); } - "!=" { return symbol(ChocoPyTokens.NEQ); } - ">=" { return symbol(ChocoPyTokens.GEQ); } - "<=" { return symbol(ChocoPyTokens.LEQ); } - "=" { return symbol(ChocoPyTokens.ASSIGN); } - "and" { return symbol(ChocoPyTokens.AND); } - "or" { return symbol(ChocoPyTokens.OR); } - "not" { return symbol(ChocoPyTokens.NOT); } - "." { return symbol(ChocoPyTokens.DOT); } - "(" { return symbol(ChocoPyTokens.LPAR); } - ")" { return symbol(ChocoPyTokens.RPAR); } - "[" { return symbol(ChocoPyTokens.LBR); } - "]" { return symbol(ChocoPyTokens.RBR); } - "->" { return symbol(ChocoPyTokens.ARROW); } - "is" { return symbol(ChocoPyTokens.IS); } - - - /*Identifiers*/ - {Names} {return symbol(ChocoPyTokens.NAMES, yytext());} + "+" { return symbol(ChocoPyTokens.PLUS, yytext()); } + /* Whitespace. */ {WhiteSpace} { /* ignore */ } - /* Comment. */ - {Comments} { /* ignore */ } -} -{ - {StringLiteral} {currString+=yytext();} - "\"" {yybegin(AFTER); return symbol(ChocoPyTokens.STRING, currString);} } + <> { return symbol(ChocoPyTokens.EOF); } /* Error fallback. */ diff --git a/src/main/jflex/chocopy/pa1/ChocoPyLexer.java b/src/main/jflex/chocopy/pa1/ChocoPyLexer.java deleted file mode 100644 index 2045209..0000000 --- a/src/main/jflex/chocopy/pa1/ChocoPyLexer.java +++ /dev/null @@ -1,1260 +0,0 @@ -// DO NOT EDIT -// Generated by JFlex 1.8.2 http://jflex.de/ -// source: ChocoPy.jflex - -package chocopy.pa1; -import java_cup.runtime.*; -import java.util.ArrayList; - -// See https://github.com/jflex-de/jflex/issues/222 -@SuppressWarnings("FallThrough") -public class ChocoPyLexer implements java_cup.runtime.Scanner { - - /** This character denotes the end of file. */ - public static final int YYEOF = -1; - - /** Initial size of the lookahead buffer. */ - private static final int ZZ_BUFFERSIZE = 16384; - - // Lexical states. - public static final int YYINITIAL = 0; - public static final int AFTER = 2; - - /** - * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l - * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l - * at the beginning of a line - * l is of the form l = 2*k, k a non negative integer - */ - private static final int ZZ_LEXSTATE[] = { - 0, 0, 1, 1 - }; - - /** - * Top-level table for translating characters to character classes - */ - private static final int [] ZZ_CMAP_TOP = zzUnpackcmap_top(); - - private static final String ZZ_CMAP_TOP_PACKED_0 = - "\1\0\u10ff\u0100"; - - private static int [] zzUnpackcmap_top() { - int [] result = new int[4352]; - int offset = 0; - offset = zzUnpackcmap_top(ZZ_CMAP_TOP_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackcmap_top(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - - /** - * Second-level tables for translating characters to character classes - */ - private static final int [] ZZ_CMAP_BLOCKS = zzUnpackcmap_blocks(); - - private static final String ZZ_CMAP_BLOCKS_PACKED_0 = - "\11\0\1\1\1\2\2\0\1\3\22\0\1\1\1\4"+ - "\1\5\1\6\1\0\1\7\2\0\1\10\1\11\1\12"+ - "\1\13\1\14\1\15\1\16\1\17\1\20\11\21\1\22"+ - "\1\0\1\23\1\24\1\25\2\0\5\26\1\27\7\26"+ - "\1\30\5\26\1\31\6\26\1\32\1\33\1\34\1\0"+ - "\1\26\1\0\1\35\1\36\1\37\1\40\1\41\1\42"+ - "\1\43\1\44\1\45\1\26\1\46\1\47\1\50\1\51"+ - "\1\52\1\53\1\26\1\54\1\55\1\56\1\57\1\26"+ - "\1\60\1\61\1\62\1\26\u0185\0"; - - private static int [] zzUnpackcmap_blocks() { - int [] result = new int[512]; - int offset = 0; - offset = zzUnpackcmap_blocks(ZZ_CMAP_BLOCKS_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackcmap_blocks(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - /** - * Translates DFA states to action switch labels. - */ - private static final int [] ZZ_ACTION = zzUnpackAction(); - - private static final String ZZ_ACTION_PACKED_0 = - "\2\0\1\1\1\2\2\3\1\4\1\5\1\6\2\7"+ - "\2\5\1\6\1\10\1\11\1\12\1\13\1\14\1\15"+ - "\1\16\1\17\1\20\2\21\1\22\1\23\1\24\1\25"+ - "\4\26\1\27\1\30\20\26\1\31\1\0\1\32\1\0"+ - "\1\33\1\20\1\34\1\35\1\36\4\26\1\37\13\26"+ - "\1\40\1\26\1\41\1\42\2\26\1\43\12\26\1\44"+ - "\5\26\1\45\1\46\4\26\1\47\5\26\1\50\3\26"+ - "\1\51\4\26\1\52\1\53\5\26\1\54\1\55\2\26"+ - "\1\56\4\26\1\57\3\26\1\60\1\26\1\61\1\26"+ - "\1\62\1\63\1\64\7\26\1\65\1\26\1\66\1\67"+ - "\1\70\1\26\1\71\1\26\1\72\1\73\1\74\1\26"+ - "\1\75\1\26\1\76\1\26\1\77\1\100"; - - private static int [] zzUnpackAction() { - int [] result = new int[172]; - int offset = 0; - offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackAction(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - - /** - * Translates a state to a row index in the transition table - */ - private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); - - private static final String ZZ_ROWMAP_PACKED_0 = - "\0\0\0\63\0\146\0\146\0\146\0\231\0\314\0\146"+ - "\0\146\0\146\0\377\0\u0132\0\u0165\0\u0198\0\146\0\146"+ - "\0\146\0\146\0\146\0\146\0\u01cb\0\146\0\u01fe\0\146"+ - "\0\u0231\0\146\0\u0264\0\u0297\0\u02ca\0\u02fd\0\u0330\0\u0363"+ - "\0\u0396\0\146\0\146\0\u03c9\0\u03fc\0\u042f\0\u0462\0\u0495"+ - "\0\u04c8\0\u04fb\0\u052e\0\u0561\0\u0594\0\u05c7\0\u05fa\0\u062d"+ - "\0\u0660\0\u0693\0\u06c6\0\146\0\u0165\0\146\0\u06f9\0\146"+ - "\0\146\0\146\0\146\0\146\0\u072c\0\u075f\0\u0792\0\u07c5"+ - "\0\u07f8\0\u082b\0\u085e\0\u0891\0\u08c4\0\u08f7\0\u092a\0\u095d"+ - "\0\u0990\0\u09c3\0\u09f6\0\u0a29\0\u02fd\0\u0a5c\0\u02fd\0\u02fd"+ - "\0\u0a8f\0\u0ac2\0\u02fd\0\u0af5\0\u0b28\0\u0b5b\0\u0b8e\0\u0bc1"+ - "\0\u0bf4\0\u0c27\0\u0c5a\0\u0c8d\0\u0cc0\0\u02fd\0\u0cf3\0\u0d26"+ - "\0\u0d59\0\u0d8c\0\u0dbf\0\u02fd\0\u02fd\0\u0df2\0\u0e25\0\u0e58"+ - "\0\u0e8b\0\u02fd\0\u0ebe\0\u0ef1\0\u0f24\0\u0f57\0\u0f8a\0\u02fd"+ - "\0\u0fbd\0\u0ff0\0\u1023\0\u02fd\0\u1056\0\u1089\0\u10bc\0\u10ef"+ - "\0\u02fd\0\u02fd\0\u1122\0\u1155\0\u1188\0\u11bb\0\u11ee\0\u02fd"+ - "\0\u02fd\0\u1221\0\u1254\0\u02fd\0\u1287\0\u12ba\0\u12ed\0\u1320"+ - "\0\u02fd\0\u1353\0\u1386\0\u13b9\0\u02fd\0\u13ec\0\u02fd\0\u141f"+ - "\0\u02fd\0\u02fd\0\u02fd\0\u1452\0\u1485\0\u14b8\0\u14eb\0\u151e"+ - "\0\u1551\0\u1584\0\u02fd\0\u15b7\0\u02fd\0\u02fd\0\u02fd\0\u15ea"+ - "\0\u02fd\0\u161d\0\u02fd\0\u02fd\0\u02fd\0\u1650\0\u02fd\0\u1683"+ - "\0\u02fd\0\u16b6\0\u02fd\0\u02fd"; - - private static int [] zzUnpackRowMap() { - int [] result = new int[172]; - int offset = 0; - offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackRowMap(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int high = packed.charAt(i++) << 16; - result[j++] = high | packed.charAt(i++); - } - return j; - } - - /** - * The transition table of the DFA - */ - private static final int [] ZZ_TRANS = zzUnpackTrans(); - - private static final String ZZ_TRANS_PACKED_0 = - "\1\3\1\4\1\5\1\6\2\3\1\7\54\3\1\10"+ - "\1\11\1\12\1\13\1\14\1\15\1\16\1\17\1\20"+ - "\1\21\1\22\1\23\1\24\1\25\1\26\1\27\1\30"+ - "\1\31\1\32\1\33\1\34\1\35\1\36\1\37\1\40"+ - "\1\41\1\42\1\10\1\43\1\44\1\45\1\46\1\47"+ - "\1\50\1\51\1\52\1\36\1\53\1\36\1\54\1\36"+ - "\1\55\1\56\1\57\1\60\1\36\1\61\1\36\1\62"+ - "\1\36\1\63\65\0\1\5\60\0\2\7\2\0\57\7"+ - "\2\0\1\12\104\0\1\64\36\0\5\65\1\66\25\65"+ - "\1\67\27\65\2\16\2\0\57\16\25\0\1\70\54\0"+ - "\1\71\63\0\2\31\65\0\1\72\62\0\1\73\62\0"+ - "\1\74\64\0\4\36\3\0\26\36\26\0\4\36\3\0"+ - "\1\75\25\36\26\0\4\36\3\0\15\36\1\76\10\36"+ - "\26\0\4\36\3\0\17\36\1\77\6\36\26\0\4\36"+ - "\3\0\14\36\1\100\3\36\1\101\2\36\1\102\2\36"+ - "\26\0\4\36\3\0\17\36\1\103\6\36\26\0\4\36"+ - "\3\0\12\36\1\104\2\36\1\105\10\36\26\0\4\36"+ - "\3\0\4\36\1\106\21\36\26\0\4\36\3\0\12\36"+ - "\1\107\11\36\1\110\1\36\26\0\4\36\3\0\10\36"+ - "\1\111\4\36\1\112\1\36\1\113\6\36\26\0\4\36"+ - "\3\0\12\36\1\114\13\36\26\0\4\36\3\0\5\36"+ - "\1\115\5\36\1\116\1\117\3\36\1\120\5\36\26\0"+ - "\4\36\3\0\1\121\25\36\26\0\4\36\3\0\15\36"+ - "\1\122\10\36\26\0\4\36\3\0\17\36\1\123\6\36"+ - "\26\0\4\36\3\0\1\124\25\36\26\0\4\36\3\0"+ - "\1\125\3\36\1\126\21\36\26\0\4\36\3\0\17\36"+ - "\1\127\6\36\26\0\4\36\3\0\7\36\1\130\1\131"+ - "\15\36\26\0\4\36\3\0\10\36\1\132\15\36\5\0"+ - "\1\65\25\0\1\65\15\0\1\65\2\0\1\65\1\0"+ - "\1\65\32\0\4\36\3\0\12\36\1\133\13\36\26\0"+ - "\4\36\3\0\14\36\1\134\11\36\26\0\4\36\3\0"+ - "\22\36\1\135\3\36\26\0\4\36\3\0\3\36\1\136"+ - "\22\36\26\0\4\36\3\0\20\36\1\137\5\36\26\0"+ - "\4\36\3\0\1\140\25\36\26\0\4\36\3\0\4\36"+ - "\1\141\21\36\26\0\4\36\3\0\1\142\25\36\26\0"+ - "\4\36\3\0\14\36\1\143\11\36\26\0\4\36\3\0"+ - "\5\36\1\144\4\36\1\145\13\36\26\0\4\36\3\0"+ - "\10\36\1\146\7\36\1\147\5\36\26\0\4\36\3\0"+ - "\2\36\1\150\23\36\26\0\4\36\3\0\14\36\1\151"+ - "\11\36\26\0\4\36\3\0\17\36\1\152\6\36\26\0"+ - "\4\36\3\0\15\36\1\153\10\36\26\0\4\36\3\0"+ - "\15\36\1\154\10\36\26\0\4\36\3\0\16\36\1\155"+ - "\7\36\26\0\4\36\3\0\13\36\1\156\12\36\26\0"+ - "\4\36\3\0\14\36\1\157\4\36\1\160\4\36\26\0"+ - "\4\36\3\0\20\36\1\161\5\36\26\0\4\36\3\0"+ - "\10\36\1\162\15\36\26\0\4\36\3\0\21\36\1\163"+ - "\4\36\26\0\4\36\3\0\25\36\1\164\26\0\4\36"+ - "\3\0\10\36\1\165\15\36\26\0\4\36\3\0\21\36"+ - "\1\166\4\36\26\0\4\36\3\0\4\36\1\167\21\36"+ - "\26\0\4\36\3\0\20\36\1\170\5\36\26\0\4\36"+ - "\3\0\4\36\1\171\21\36\26\0\4\36\3\0\4\36"+ - "\1\172\21\36\26\0\4\36\3\0\4\36\1\173\21\36"+ - "\26\0\4\36\3\0\10\36\1\174\15\36\26\0\4\36"+ - "\3\0\1\175\25\36\26\0\4\36\3\0\20\36\1\176"+ - "\5\36\26\0\4\36\3\0\21\36\1\177\4\36\26\0"+ - "\4\36\3\0\5\36\1\200\20\36\26\0\4\36\3\0"+ - "\4\36\1\201\21\36\26\0\4\36\3\0\4\36\1\202"+ - "\21\36\26\0\4\36\3\0\1\203\25\36\26\0\4\36"+ - "\3\0\13\36\1\204\12\36\26\0\4\36\3\0\1\36"+ - "\1\205\24\36\26\0\4\36\3\0\15\36\1\206\10\36"+ - "\26\0\4\36\3\0\1\36\1\207\24\36\26\0\4\36"+ - "\3\0\12\36\1\210\13\36\26\0\4\36\3\0\20\36"+ - "\1\211\5\36\26\0\4\36\3\0\20\36\1\212\5\36"+ - "\26\0\4\36\3\0\22\36\1\213\3\36\26\0\4\36"+ - "\3\0\12\36\1\214\13\36\26\0\4\36\3\0\7\36"+ - "\1\215\16\36\26\0\4\36\3\0\12\36\1\216\13\36"+ - "\26\0\4\36\3\0\4\36\1\217\21\36\26\0\4\36"+ - "\3\0\17\36\1\220\6\36\26\0\4\36\3\0\21\36"+ - "\1\221\4\36\26\0\4\36\3\0\11\36\1\222\14\36"+ - "\26\0\4\36\3\0\20\36\1\223\5\36\26\0\4\36"+ - "\3\0\10\36\1\224\15\36\26\0\4\36\3\0\16\36"+ - "\1\225\7\36\26\0\4\36\3\0\12\36\1\226\13\36"+ - "\26\0\4\36\3\0\1\227\25\36\26\0\4\36\3\0"+ - "\17\36\1\230\6\36\26\0\4\36\3\0\3\36\1\231"+ - "\22\36\26\0\4\36\3\0\15\36\1\232\10\36\26\0"+ - "\4\36\3\0\4\36\1\233\21\36\26\0\4\36\3\0"+ - "\17\36\1\234\6\36\26\0\4\36\3\0\4\36\1\235"+ - "\21\36\26\0\4\36\3\0\3\36\1\236\22\36\26\0"+ - "\4\36\3\0\21\36\1\237\4\36\26\0\4\36\3\0"+ - "\14\36\1\240\11\36\26\0\4\36\3\0\21\36\1\241"+ - "\4\36\26\0\4\36\3\0\12\36\1\242\13\36\26\0"+ - "\4\36\3\0\12\36\1\243\13\36\26\0\4\36\3\0"+ - "\21\36\1\244\4\36\26\0\4\36\3\0\1\245\25\36"+ - "\26\0\4\36\3\0\2\36\1\246\23\36\26\0\4\36"+ - "\3\0\14\36\1\247\11\36\26\0\4\36\3\0\22\36"+ - "\1\250\3\36\26\0\4\36\3\0\25\36\1\251\26\0"+ - "\4\36\3\0\1\252\25\36\26\0\4\36\3\0\4\36"+ - "\1\253\21\36\26\0\4\36\3\0\12\36\1\254\13\36"; - - private static int [] zzUnpackTrans() { - int [] result = new int[5865]; - int offset = 0; - offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackTrans(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - value--; - do result[j++] = value; while (--count > 0); - } - return j; - } - - - /** Error code for "Unknown internal scanner error". */ - private static final int ZZ_UNKNOWN_ERROR = 0; - /** Error code for "could not match input". */ - private static final int ZZ_NO_MATCH = 1; - /** Error code for "pushback value was too large". */ - private static final int ZZ_PUSHBACK_2BIG = 2; - - /** - * Error messages for {@link #ZZ_UNKNOWN_ERROR}, {@link #ZZ_NO_MATCH}, and - * {@link #ZZ_PUSHBACK_2BIG} respectively. - */ - private static final String ZZ_ERROR_MSG[] = { - "Unknown internal scanner error", - "Error: could not match input", - "Error: pushback value was too large" - }; - - /** - * ZZ_ATTRIBUTE[aState] contains the attributes of state {@code aState} - */ - private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute(); - - private static final String ZZ_ATTRIBUTE_PACKED_0 = - "\2\0\3\11\2\1\3\11\4\1\6\11\1\1\1\11"+ - "\1\1\1\11\1\1\1\11\7\1\2\11\20\1\1\11"+ - "\1\0\1\11\1\0\5\11\160\1"; - - private static int [] zzUnpackAttribute() { - int [] result = new int[172]; - int offset = 0; - offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackAttribute(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - /** Input device. */ - private java.io.Reader zzReader; - - /** Current state of the DFA. */ - private int zzState; - - /** Current lexical state. */ - private int zzLexicalState = YYINITIAL; - - /** - * This buffer contains the current text to be matched and is the source of the {@link #yytext()} - * string. - */ - private char zzBuffer[] = new char[ZZ_BUFFERSIZE]; - - /** Text position at the last accepting state. */ - private int zzMarkedPos; - - /** Current text position in the buffer. */ - private int zzCurrentPos; - - /** Marks the beginning of the {@link #yytext()} string in the buffer. */ - private int zzStartRead; - - /** Marks the last character in the buffer, that has been read from input. */ - private int zzEndRead; - - /** - * Whether the scanner is at the end of file. - * @see #yyatEOF - */ - private boolean zzAtEOF; - - /** - * The number of occupied positions in {@link #zzBuffer} beyond {@link #zzEndRead}. - * - *

When a lead/high surrogate has been read from the input stream into the final - * {@link #zzBuffer} position, this will have a value of 1; otherwise, it will have a value of 0. - */ - private int zzFinalHighSurrogate = 0; - - /** Number of newlines encountered up to the start of the matched text. */ - private int yyline; - - /** Number of characters from the last newline up to the start of the matched text. */ - private int yycolumn; - - /** Number of characters up to the start of the matched text. */ - @SuppressWarnings("unused") - private long yychar; - - /** Whether the scanner is currently at the beginning of a line. */ - @SuppressWarnings("unused") - private boolean zzAtBOL = true; - - /** Whether the user-EOF-code has already been executed. */ - @SuppressWarnings("unused") - private boolean zzEOFDone; - - /* user code: */ - /* The code below includes some convenience methods to create tokens - * of a given type and optionally a value that the CUP parser can - * understand. Specifically, a lot of the logic below deals with - * embedded information about where in the source code a given token - * was recognized, so that the parser can report errors accurately. - * (It need not be modified for this project.) */ - - /** Producer of token-related values for the parser. */ - final ComplexSymbolFactory symbolFactory = new ComplexSymbolFactory(); - private int currIndent = 0; - private ArrayList stack = new ArrayList(20); - /** Return a terminal symbol of syntactic category TYPE and no - * semantic value at the current source location. */ - private Symbol symbol(int type) { - return symbol(type); - } - - /** Return a terminal symbol of syntactic category TYPE and semantic - * value VALUE at the current source location. */ - private Symbol symbol(int type, Object value) { - return symbolFactory.newSymbol(ChocoPyTokens.terminalNames[type], type, - new ComplexSymbolFactory.Location(yyline + 1, yycolumn + 1), - new ComplexSymbolFactory.Location(yyline + 1,yycolumn + yylength()), - value); - } - private void push(int indent){ - stack.add(indent); - } - private int pop(){ - if(stack.isEmpty()) return 0; - return stack.remove(stack.size() - 1); - } - private int top(){ - if(stack.isEmpty) return 0; - return stack.get(stack.size() - 1); - } - - - /** - * Creates a new scanner - * - * @param in the java.io.Reader to read input from. - */ - public ChocoPyLexer(java.io.Reader in) { - this.zzReader = in; - } - - /** - * Translates raw input code points to DFA table row - */ - private static int zzCMap(int input) { - int offset = input & 255; - return offset == input ? ZZ_CMAP_BLOCKS[offset] : ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input >> 8] | offset]; - } - - /** - * Refills the input buffer. - * - * @return {@code false} iff there was new input. - * @exception java.io.IOException if any I/O-Error occurs - */ - private boolean zzRefill() throws java.io.IOException { - - /* first: make room (if you can) */ - if (zzStartRead > 0) { - zzEndRead += zzFinalHighSurrogate; - zzFinalHighSurrogate = 0; - System.arraycopy(zzBuffer, zzStartRead, - zzBuffer, 0, - zzEndRead - zzStartRead); - - /* translate stored positions */ - zzEndRead -= zzStartRead; - zzCurrentPos -= zzStartRead; - zzMarkedPos -= zzStartRead; - zzStartRead = 0; - } - - /* is the buffer big enough? */ - if (zzCurrentPos >= zzBuffer.length - zzFinalHighSurrogate) { - /* if not: blow it up */ - char newBuffer[] = new char[zzBuffer.length * 2]; - System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length); - zzBuffer = newBuffer; - zzEndRead += zzFinalHighSurrogate; - zzFinalHighSurrogate = 0; - } - - /* fill the buffer with new input */ - int requested = zzBuffer.length - zzEndRead; - int numRead = zzReader.read(zzBuffer, zzEndRead, requested); - - /* not supposed to occur according to specification of java.io.Reader */ - if (numRead == 0) { - throw new java.io.IOException( - "Reader returned 0 characters. See JFlex examples/zero-reader for a workaround."); - } - if (numRead > 0) { - zzEndRead += numRead; - if (Character.isHighSurrogate(zzBuffer[zzEndRead - 1])) { - if (numRead == requested) { // We requested too few chars to encode a full Unicode character - --zzEndRead; - zzFinalHighSurrogate = 1; - } else { // There is room in the buffer for at least one more char - int c = zzReader.read(); // Expecting to read a paired low surrogate char - if (c == -1) { - return true; - } else { - zzBuffer[zzEndRead++] = (char)c; - } - } - } - /* potentially more input available */ - return false; - } - - /* numRead < 0 ==> end of stream */ - return true; - } - - - /** - * Closes the input reader. - * - * @throws java.io.IOException if the reader could not be closed. - */ - public final void yyclose() throws java.io.IOException { - zzAtEOF = true; // indicate end of file - zzEndRead = zzStartRead; // invalidate buffer - - if (zzReader != null) { - zzReader.close(); - } - } - - - /** - * Resets the scanner to read from a new input stream. - * - *

Does not close the old reader. - * - *

All internal variables are reset, the old input stream cannot be reused (internal - * buffer is discarded and lost). Lexical state is set to {@code ZZ_INITIAL}. - * - *

Internal scan buffer is resized down to its initial length, if it has grown. - * - * @param reader The new input stream. - */ - public final void yyreset(java.io.Reader reader) { - zzReader = reader; - zzEOFDone = false; - yyResetPosition(); - zzLexicalState = YYINITIAL; - if (zzBuffer.length > ZZ_BUFFERSIZE) { - zzBuffer = new char[ZZ_BUFFERSIZE]; - } - } - - /** - * Resets the input position. - */ - private final void yyResetPosition() { - zzAtBOL = true; - zzAtEOF = false; - zzCurrentPos = 0; - zzMarkedPos = 0; - zzStartRead = 0; - zzEndRead = 0; - zzFinalHighSurrogate = 0; - yyline = 0; - yycolumn = 0; - yychar = 0L; - } - - - /** - * Returns whether the scanner has reached the end of the reader it reads from. - * - * @return whether the scanner has reached EOF. - */ - public final boolean yyatEOF() { - return zzAtEOF; - } - - - /** - * Returns the current lexical state. - * - * @return the current lexical state. - */ - public final int yystate() { - return zzLexicalState; - } - - - /** - * Enters a new lexical state. - * - * @param newState the new lexical state - */ - public final void yybegin(int newState) { - zzLexicalState = newState; - } - - - /** - * Returns the text matched by the current regular expression. - * - * @return the matched text. - */ - public final String yytext() { - return new String(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); - } - - - /** - * Returns the character at the given position from the matched text. - * - *

It is equivalent to {@code yytext().charAt(pos)}, but faster. - * - * @param position the position of the character to fetch. A value from 0 to {@code yylength()-1}. - * - * @return the character at {@code position}. - */ - public final char yycharat(int position) { - return zzBuffer[zzStartRead + position]; - } - - - /** - * How many characters were matched. - * - * @return the length of the matched text region. - */ - public final int yylength() { - return zzMarkedPos-zzStartRead; - } - - - /** - * Reports an error that occurred while scanning. - * - *

In a well-formed scanner (no or only correct usage of {@code yypushback(int)} and a - * match-all fallback rule) this method will only be called with things that - * "Can't Possibly Happen". - * - *

If this method is called, something is seriously wrong (e.g. a JFlex bug producing a faulty - * scanner etc.). - * - *

Usual syntax/scanner level error handling should be done in error fallback rules. - * - * @param errorCode the code of the error message to display. - */ - private static void zzScanError(int errorCode) { - String message; - try { - message = ZZ_ERROR_MSG[errorCode]; - } catch (ArrayIndexOutOfBoundsException e) { - message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; - } - - throw new Error(message); - } - - - /** - * Pushes the specified amount of characters back into the input stream. - * - *

They will be read again by then next call of the scanning method. - * - * @param number the number of characters to be read again. This number must not be greater than - * {@link #yylength()}. - */ - public void yypushback(int number) { - if ( number > yylength() ) - zzScanError(ZZ_PUSHBACK_2BIG); - - zzMarkedPos -= number; - } - - - - - /** - * Resumes scanning until the next regular expression is matched, the end of input is encountered - * or an I/O-Error occurs. - * - * @return the next token. - * @exception java.io.IOException if any I/O-Error occurs. - */ - @Override public java_cup.runtime.Symbol next_token() throws java.io.IOException { - int zzInput; - int zzAction; - - // cached fields: - int zzCurrentPosL; - int zzMarkedPosL; - int zzEndReadL = zzEndRead; - char[] zzBufferL = zzBuffer; - - int [] zzTransL = ZZ_TRANS; - int [] zzRowMapL = ZZ_ROWMAP; - int [] zzAttrL = ZZ_ATTRIBUTE; - - while (true) { - zzMarkedPosL = zzMarkedPos; - - boolean zzR = false; - int zzCh; - int zzCharCount; - for (zzCurrentPosL = zzStartRead ; - zzCurrentPosL < zzMarkedPosL ; - zzCurrentPosL += zzCharCount ) { - zzCh = Character.codePointAt(zzBufferL, zzCurrentPosL, zzMarkedPosL); - zzCharCount = Character.charCount(zzCh); - switch (zzCh) { - case '\u000B': // fall through - case '\u000C': // fall through - case '\u0085': // fall through - case '\u2028': // fall through - case '\u2029': - yyline++; - yycolumn = 0; - zzR = false; - break; - case '\r': - yyline++; - yycolumn = 0; - zzR = true; - break; - case '\n': - if (zzR) - zzR = false; - else { - yyline++; - yycolumn = 0; - } - break; - default: - zzR = false; - yycolumn += zzCharCount; - } - } - - if (zzR) { - // peek one character ahead if it is - // (if we have counted one line too much) - boolean zzPeek; - if (zzMarkedPosL < zzEndReadL) - zzPeek = zzBufferL[zzMarkedPosL] == '\n'; - else if (zzAtEOF) - zzPeek = false; - else { - boolean eof = zzRefill(); - zzEndReadL = zzEndRead; - zzMarkedPosL = zzMarkedPos; - zzBufferL = zzBuffer; - if (eof) - zzPeek = false; - else - zzPeek = zzBufferL[zzMarkedPosL] == '\n'; - } - if (zzPeek) yyline--; - } - zzAction = -1; - - zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL; - - zzState = ZZ_LEXSTATE[zzLexicalState]; - - // set up zzAction for empty match case: - int zzAttributes = zzAttrL[zzState]; - if ( (zzAttributes & 1) == 1 ) { - zzAction = zzState; - } - - - zzForAction: { - while (true) { - - if (zzCurrentPosL < zzEndReadL) { - zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL); - zzCurrentPosL += Character.charCount(zzInput); - } - else if (zzAtEOF) { - zzInput = YYEOF; - break zzForAction; - } - else { - // store back cached positions - zzCurrentPos = zzCurrentPosL; - zzMarkedPos = zzMarkedPosL; - boolean eof = zzRefill(); - // get translated positions and possibly new buffer - zzCurrentPosL = zzCurrentPos; - zzMarkedPosL = zzMarkedPos; - zzBufferL = zzBuffer; - zzEndReadL = zzEndRead; - if (eof) { - zzInput = YYEOF; - break zzForAction; - } - else { - zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL); - zzCurrentPosL += Character.charCount(zzInput); - } - } - int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMap(zzInput) ]; - if (zzNext == -1) break zzForAction; - zzState = zzNext; - - zzAttributes = zzAttrL[zzState]; - if ( (zzAttributes & 1) == 1 ) { - zzAction = zzState; - zzMarkedPosL = zzCurrentPosL; - if ( (zzAttributes & 8) == 8 ) break zzForAction; - } - - } - } - - // store back cached position - zzMarkedPos = zzMarkedPosL; - - if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { - zzAtEOF = true; - { - return symbol(ChocoPyTokens.EOF); - } - } - else { - switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) { - case 1: - { yypushback(1); - if(top() > currIndent) - { - pop(); - return symbol(ChocoPyTokens.DEDENT); - } - yystart(AFTER); - if(top()< currIndent) - { - push(currIndent); - return symbol(ChocoPyTokens.INDENT); - } - } - // fall through - case 65: break; - case 2: - { String space = yytext(); - if(space == "\t") - currIndent += 8; - else - currIndent ++; - } - // fall through - case 66: break; - case 3: - { currIndent = 0; - } - // fall through - case 67: break; - case 4: - { /* ignored */ - } - // fall through - case 68: break; - case 5: - { return symbol(ChocoPyTokens.UNRECOGNIZED); - } - // fall through - case 69: break; - case 6: - { /* ignore */ - } - // fall through - case 70: break; - case 7: - { return symbol(ChocoPyTokens.NEWLINE); yybegin(YYINITIAL); currIndent = 0; - } - // fall through - case 71: break; - case 8: - { return symbol(ChocoPyTokens.MOD); - } - // fall through - case 72: break; - case 9: - { return symbol(ChocoPyTokens.LPAR); - } - // fall through - case 73: break; - case 10: - { return symbol(ChocoPyTokens.RPAR); - } - // fall through - case 74: break; - case 11: - { return symbol(ChocoPyTokens.MUL); - } - // fall through - case 75: break; - case 12: - { return symbol(ChocoPyTokens.PLUS); - } - // fall through - case 76: break; - case 13: - { return symbol(ChocoPyTokens.COMMA); - } - // fall through - case 77: break; - case 14: - { return symbol(ChocoPyTokens.MINUS); - } - // fall through - case 78: break; - case 15: - { return symbol(ChocoPyTokens.DOT); - } - // fall through - case 79: break; - case 16: - { return symbol(ChocoPyTokens.DIV); - } - // fall through - case 80: break; - case 17: - { return symbol(ChocoPyTokens.NUMBER, - Integer.parseInt(yytext())); - } - // fall through - case 81: break; - case 18: - { return symbol(ChocoPyTokens.COLON); - } - // fall through - case 82: break; - case 19: - { return symbol(ChocoPyTokens.LT); - } - // fall through - case 83: break; - case 20: - { return symbol(ChocoPyTokens.ASSIGN); - } - // fall through - case 84: break; - case 21: - { return symbol(ChocoPyTokens.GT); - } - // fall through - case 85: break; - case 22: - { return symbol(ChocoPyTokens.NAMES, yytext()); - } - // fall through - case 86: break; - case 23: - { return symbol(ChocoPyTokens.LBR); - } - // fall through - case 87: break; - case 24: - { return symbol(ChocoPyTokens.RBR); - } - // fall through - case 88: break; - case 25: - { return symbol(ChocoPyTokens.NEQ); - } - // fall through - case 89: break; - case 26: - { return symbol(ChocoPyTokens.STRING, yytext()); - } - // fall through - case 90: break; - case 27: - { return symbol(ChocoPyTokens.ARROW); - } - // fall through - case 91: break; - case 28: - { return symbol(ChocoPyTokens.LEQ); - } - // fall through - case 92: break; - case 29: - { return symbol(ChocoPyTokens.EQUAL); - } - // fall through - case 93: break; - case 30: - { return symbol(ChocoPyTokens.GEQ); - } - // fall through - case 94: break; - case 31: - { return symbol(ChocoPyTokens.AS); - } - // fall through - case 95: break; - case 32: - { return symbol(ChocoPyTokens.IF); - } - // fall through - case 96: break; - case 33: - { return symbol(ChocoPyTokens.IN); - } - // fall through - case 97: break; - case 34: - { return symbol(ChocoPyTokens.IS); - } - // fall through - case 98: break; - case 35: - { return symbol(ChocoPyTokens.OR); - } - // fall through - case 99: break; - case 36: - { return symbol(ChocoPyTokens.AND); - } - // fall through - case 100: break; - case 37: - { return symbol(ChocoPyTokens.DEF); - } - // fall through - case 101: break; - case 38: - { return symbol(ChocoPyTokens.DEL); - } - // fall through - case 102: break; - case 39: - { return symbol(ChocoPyTokens.FOR); - } - // fall through - case 103: break; - case 40: - { return symbol(ChocoPyTokens.NOT); - } - // fall through - case 104: break; - case 41: - { return symbol(ChocoPyTokens.TRY); - } - // fall through - case 105: break; - case 42: - { return symbol(ChocoPyTokens.NONE); - } - // fall through - case 106: break; - case 43: - { return symbol(ChocoPyTokens.BOOL, true); - } - // fall through - case 107: break; - case 44: - { return symbol(ChocoPyTokens.ELIF); - } - // fall through - case 108: break; - case 45: - { return symbol(ChocoPyTokens.ELSE); - } - // fall through - case 109: break; - case 46: - { return symbol(ChocoPyTokens.FROM); - } - // fall through - case 110: break; - case 47: - { return symbol(ChocoPyTokens.PASS); - } - // fall through - case 111: break; - case 48: - { return symbol(ChocoPyTokens.WITH); - } - // fall through - case 112: break; - case 49: - { return symbol(ChocoPyTokens.BOOL, false); - } - // fall through - case 113: break; - case 50: - { return symbol(ChocoPyTokens.AWAIT); - } - // fall through - case 114: break; - case 51: - { return symbol(ChocoPyTokens.BREAK); - } - // fall through - case 115: break; - case 52: - { return symbol(ChocoPyTokens.CLASS); - } - // fall through - case 116: break; - case 53: - { return symbol(ChocoPyTokens.RAISE); - } - // fall through - case 117: break; - case 54: - { return symbol(ChocoPyTokens.WHILE); - } - // fall through - case 118: break; - case 55: - { return symbol(ChocoPyTokens.YIELD); - } - // fall through - case 119: break; - case 56: - { return symbol(ChocoPyTokens.ASSERT); - } - // fall through - case 120: break; - case 57: - { return symbol(ChocoPyTokens.EXCEPT); - } - // fall through - case 121: break; - case 58: - { return symbol(ChocoPyTokens.GLOBAL); - } - // fall through - case 122: break; - case 59: - { return symbol(ChocoPyTokens.IMPORT); - } - // fall through - case 123: break; - case 60: - { return symbol(ChocoPyTokens.LAMBDA); - } - // fall through - case 124: break; - case 61: - { return symbol(ChocoPyTokens.RETURN); - } - // fall through - case 125: break; - case 62: - { return symbol(ChocoPyTokens.FINALLY); - } - // fall through - case 126: break; - case 63: - { return symbol(ChocoPyTokens.CONTINUE); - } - // fall through - case 127: break; - case 64: - { return symbol(ChocoPyTokens.NONLOCAL); - } - // fall through - case 128: break; - default: - zzScanError(ZZ_NO_MATCH); - } - } - } - } - - /** - * Converts an int token code into the name of the - * token by reflection on the cup symbol class/interface ChocoPyTokens - */ - private static String getTokenName(int token) { - try { - java.lang.reflect.Field [] classFields = ChocoPyTokens.class.getFields(); - for (int i = 0; i < classFields.length; i++) { - if (classFields[i].getInt(null) == token) { - return classFields[i].getName(); - } - } - } catch (Exception e) { - e.printStackTrace(System.err); - } - - return "UNKNOWN TOKEN"; - } - - /** - * Same as next_token but also prints the token to standard out - * for debugging. - */ - public java_cup.runtime.Symbol debug_next_token() throws java.io.IOException { - java_cup.runtime.Symbol s = next_token(); - System.out.println( "line:" + (yyline+1) + " col:" + (yycolumn+1) + " --"+ yytext() + "--" + getTokenName(s.sym) + "--"); - return s; - } - - /** - * Runs the scanner on input files. - * - * This main method is the debugging routine for the scanner. - * It prints debugging information about each returned token to - * System.out until the end of file is reached, or an error occured. - * - * @param argv the command line, contains the filenames to run - * the scanner on. - */ - public static void main(String[] argv) { - if (argv.length == 0) { - System.out.println("Usage : java ChocoPyLexer [ --encoding ] "); - } - else { - int firstFilePos = 0; - String encodingName = "UTF-8"; - if (argv[0].equals("--encoding")) { - firstFilePos = 2; - encodingName = argv[1]; - try { - // Side-effect: is encodingName valid? - java.nio.charset.Charset.forName(encodingName); - } catch (Exception e) { - System.out.println("Invalid encoding '" + encodingName + "'"); - return; - } - } - for (int i = firstFilePos; i < argv.length; i++) { - ChocoPyLexer scanner = null; - try { - java.io.FileInputStream stream = new java.io.FileInputStream(argv[i]); - java.io.Reader reader = new java.io.InputStreamReader(stream, encodingName); - scanner = new ChocoPyLexer(reader); - while ( !scanner.zzAtEOF ) scanner.debug_next_token(); - } - catch (java.io.FileNotFoundException e) { - System.out.println("File not found : \""+argv[i]+"\""); - } - catch (java.io.IOException e) { - System.out.println("IO error scanning file \""+argv[i]+"\""); - System.out.println(e); - } - catch (Exception e) { - System.out.println("Unexpected exception:"); - e.printStackTrace(); - } - } - } - } - - -}