From ecb8d23418dd389b5160205022981e36b2c54425 Mon Sep 17 00:00:00 2001 From: bill Date: Thu, 18 Feb 2021 00:36:30 +0800 Subject: [PATCH] bug fixes --- src/main/jflex/chocopy/pa1/ChocoPy.jflex | 8 +- src/main/jflex/chocopy/pa1/ChocoPyLexer.java | 1287 ------------------ 2 files changed, 4 insertions(+), 1291 deletions(-) delete mode 100644 src/main/jflex/chocopy/pa1/ChocoPyLexer.java diff --git a/src/main/jflex/chocopy/pa1/ChocoPy.jflex b/src/main/jflex/chocopy/pa1/ChocoPy.jflex index dc0677f..ccb7187 100644 --- a/src/main/jflex/chocopy/pa1/ChocoPy.jflex +++ b/src/main/jflex/chocopy/pa1/ChocoPy.jflex @@ -33,7 +33,7 @@ import java.util.ArrayList; /** Producer of token-related values for the parser. */ final ComplexSymbolFactory symbolFactory = new ComplexSymbolFactory(); private int currIndent = 0; //Current Indentation Level - private int currString = ""; + private String currString = ""; /*A stack that keeps track of the spaces in each Indentation Level*/ private ArrayList stack = new ArrayList(20); /** Return a terminal symbol of syntactic category TYPE and no @@ -70,7 +70,7 @@ WhiteSpace = [ \t] LineBreak = \r|\n|\r\n IntegerLiteral = 0|[1-9][0-9]* // Accroding to the manual, 00+ is illeagal -StringLiteral = ([^\"\\]|(\\\")|(\\t)|(\\r)|(\\n)|(\\\\))* // \n, \r, \t, \\, \" and Anything except \ and " +StringLiteral = ([^\"\\]|(\\\")|(\\t)|(\\r)|(\\n)|(\\\\))+ // \n, \r, \t, \\, \" and Anything except \ and " Identifiers = (_|[a-z]|[A-Z])(_|[a-z]|[A-Z][0-9])* Comments = #[^\r\n]* %% @@ -138,7 +138,7 @@ if True: } /*Otherwise, we will start dealing with the rest of the line after indentation in AFTER state. */ - yystart(AFTER); + yybegin(AFTER); if(top()< currIndent) { /* @@ -160,7 +160,7 @@ if True: /* Literals. */ {IntegerLiteral} { return symbol(ChocoPyTokens.NUMBER, Integer.parseInt(yytext())); } - "\"" {yystart(STR); currString = "";} //Start taking a string when see a " + "\"" {yybegin(STR); currString = "";} //Start taking a string when see a " "False" { return symbol(ChocoPyTokens.BOOL, false); } "True" { return symbol(ChocoPyTokens.BOOL, true); } "None" { return symbol(ChocoPyTokens.NONE); } diff --git a/src/main/jflex/chocopy/pa1/ChocoPyLexer.java b/src/main/jflex/chocopy/pa1/ChocoPyLexer.java deleted file mode 100644 index ac26ed7..0000000 --- a/src/main/jflex/chocopy/pa1/ChocoPyLexer.java +++ /dev/null @@ -1,1287 +0,0 @@ -// DO NOT EDIT -// Generated by JFlex 1.8.2 http://jflex.de/ -// source: ChocoPy.jflex - -package chocopy.pa1; -import java_cup.runtime.*; -import java.util.ArrayList; - -// See https://github.com/jflex-de/jflex/issues/222 -@SuppressWarnings("FallThrough") -public class ChocoPyLexer implements java_cup.runtime.Scanner { - - /** This character denotes the end of file. */ - public static final int YYEOF = -1; - - /** Initial size of the lookahead buffer. */ - private static final int ZZ_BUFFERSIZE = 16384; - - // Lexical states. - public static final int YYINITIAL = 0; - public static final int AFTER = 2; - public static final int STR = 4; - - /** - * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l - * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l - * at the beginning of a line - * l is of the form l = 2*k, k a non negative integer - */ - private static final int ZZ_LEXSTATE[] = { - 0, 0, 1, 1, 2, 2 - }; - - /** - * Top-level table for translating characters to character classes - */ - private static final int [] ZZ_CMAP_TOP = zzUnpackcmap_top(); - - private static final String ZZ_CMAP_TOP_PACKED_0 = - "\1\0\37\u0100\1\u0200\u10df\u0100"; - - private static int [] zzUnpackcmap_top() { - int [] result = new int[4352]; - int offset = 0; - offset = zzUnpackcmap_top(ZZ_CMAP_TOP_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackcmap_top(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - - /** - * Second-level tables for translating characters to character classes - */ - private static final int [] ZZ_CMAP_BLOCKS = zzUnpackcmap_blocks(); - - private static final String ZZ_CMAP_BLOCKS_PACKED_0 = - "\11\0\1\1\1\2\2\3\1\4\22\0\1\1\1\5"+ - "\1\6\1\7\1\0\1\10\2\0\1\11\1\12\1\13"+ - "\1\14\1\15\1\16\1\17\1\20\1\21\11\22\1\23"+ - "\1\0\1\24\1\25\1\26\2\0\5\27\1\30\7\27"+ - "\1\31\5\27\1\32\6\27\1\33\1\34\1\35\1\0"+ - "\1\27\1\0\1\36\1\37\1\40\1\41\1\42\1\43"+ - "\1\44\1\45\1\46\1\27\1\47\1\50\1\51\1\52"+ - "\1\53\1\54\1\27\1\55\1\56\1\57\1\60\1\27"+ - "\1\61\1\62\1\63\1\27\12\0\1\3\u01a2\0\2\3"+ - "\326\0"; - - private static int [] zzUnpackcmap_blocks() { - int [] result = new int[768]; - int offset = 0; - offset = zzUnpackcmap_blocks(ZZ_CMAP_BLOCKS_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackcmap_blocks(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - /** - * Translates DFA states to action switch labels. - */ - private static final int [] ZZ_ACTION = zzUnpackAction(); - - private static final String ZZ_ACTION_PACKED_0 = - "\2\0\1\1\1\2\1\3\2\4\1\5\1\6\1\7"+ - "\2\10\1\6\1\11\1\7\1\12\1\13\1\14\1\15"+ - "\1\16\1\17\1\20\1\21\1\22\2\23\1\24\1\25"+ - "\1\26\1\27\4\30\1\31\1\32\20\30\1\1\1\33"+ - "\1\6\1\34\1\35\1\22\1\36\1\37\1\40\4\30"+ - "\1\41\13\30\1\42\1\30\1\43\1\44\2\30\1\45"+ - "\7\30\1\0\2\46\3\30\1\47\5\30\1\50\1\51"+ - "\4\30\1\52\5\30\1\53\3\30\1\54\4\30\1\55"+ - "\1\56\5\30\1\57\1\60\2\30\1\61\4\30\1\62"+ - "\3\30\1\63\1\30\1\64\1\30\1\65\1\66\1\67"+ - "\7\30\1\70\1\30\1\71\1\72\1\73\1\30\1\74"+ - "\1\30\1\75\1\76\1\77\1\30\1\100\1\30\1\101"+ - "\1\30\1\102\1\103"; - - private static int [] zzUnpackAction() { - int [] result = new int[176]; - int offset = 0; - offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackAction(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - - /** - * Translates a state to a row index in the transition table - */ - private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); - - private static final String ZZ_ROWMAP_PACKED_0 = - "\0\0\0\64\0\150\0\234\0\234\0\234\0\320\0\u0104"+ - "\0\234\0\234\0\234\0\u0138\0\u016c\0\234\0\u01a0\0\234"+ - "\0\234\0\234\0\234\0\234\0\234\0\u01d4\0\234\0\u0208"+ - "\0\234\0\u023c\0\234\0\u0270\0\u02a4\0\u02d8\0\u030c\0\u0340"+ - "\0\u0374\0\u03a8\0\234\0\234\0\u03dc\0\u0410\0\u0444\0\u0478"+ - "\0\u04ac\0\u04e0\0\u0514\0\u0548\0\u057c\0\u05b0\0\u05e4\0\u0618"+ - "\0\u064c\0\u0680\0\u06b4\0\u06e8\0\u071c\0\234\0\u0750\0\234"+ - "\0\234\0\234\0\234\0\234\0\234\0\u0784\0\u07b8\0\u07ec"+ - "\0\u0820\0\u0854\0\u0888\0\u08bc\0\u08f0\0\u0924\0\u0958\0\u098c"+ - "\0\u09c0\0\u09f4\0\u0a28\0\u0a5c\0\u0a90\0\u030c\0\u0ac4\0\u030c"+ - "\0\u030c\0\u0af8\0\u0b2c\0\u030c\0\u0b60\0\u0b94\0\u0bc8\0\u0bfc"+ - "\0\u0c30\0\u0c64\0\u0c98\0\u0ccc\0\234\0\u0d00\0\u0d34\0\u0d68"+ - "\0\u0d9c\0\u030c\0\u0dd0\0\u0e04\0\u0e38\0\u0e6c\0\u0ea0\0\u030c"+ - "\0\u030c\0\u0ed4\0\u0f08\0\u0f3c\0\u0f70\0\u030c\0\u0fa4\0\u0fd8"+ - "\0\u100c\0\u1040\0\u1074\0\u030c\0\u10a8\0\u10dc\0\u1110\0\u030c"+ - "\0\u1144\0\u1178\0\u11ac\0\u11e0\0\u030c\0\u030c\0\u1214\0\u1248"+ - "\0\u127c\0\u12b0\0\u12e4\0\u030c\0\u030c\0\u1318\0\u134c\0\u030c"+ - "\0\u1380\0\u13b4\0\u13e8\0\u141c\0\u030c\0\u1450\0\u1484\0\u14b8"+ - "\0\u030c\0\u14ec\0\u030c\0\u1520\0\u030c\0\u030c\0\u030c\0\u1554"+ - "\0\u1588\0\u15bc\0\u15f0\0\u1624\0\u1658\0\u168c\0\u030c\0\u16c0"+ - "\0\u030c\0\u030c\0\u030c\0\u16f4\0\u030c\0\u1728\0\u030c\0\u030c"+ - "\0\u030c\0\u175c\0\u030c\0\u1790\0\u030c\0\u17c4\0\u030c\0\u030c"; - - private static int [] zzUnpackRowMap() { - int [] result = new int[176]; - int offset = 0; - offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackRowMap(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int high = packed.charAt(i++) << 16; - result[j++] = high | packed.charAt(i++); - } - return j; - } - - /** - * The transition table of the DFA - */ - private static final int [] ZZ_TRANS = zzUnpackTrans(); - - private static final String ZZ_TRANS_PACKED_0 = - "\1\4\1\5\1\6\1\4\1\7\2\4\1\10\54\4"+ - "\1\11\1\12\1\13\1\11\1\14\1\15\1\16\1\17"+ - "\1\20\1\21\1\22\1\23\1\24\1\25\1\26\1\27"+ - "\1\30\1\31\1\32\1\33\1\34\1\35\1\36\1\37"+ - "\1\40\1\41\1\42\1\43\1\11\1\44\1\45\1\46"+ - "\1\47\1\50\1\51\1\52\1\53\1\37\1\54\1\37"+ - "\1\55\1\37\1\56\1\57\1\60\1\61\1\37\1\62"+ - "\1\37\1\63\1\37\1\64\6\65\1\66\25\65\1\67"+ - "\27\65\66\0\1\6\61\0\2\10\1\0\1\10\1\0"+ - "\57\10\2\0\1\13\106\0\1\70\36\0\2\17\1\0"+ - "\1\17\1\0\57\17\26\0\1\71\55\0\1\72\64\0"+ - "\2\32\66\0\1\73\63\0\1\74\63\0\1\75\65\0"+ - "\4\37\3\0\26\37\27\0\4\37\3\0\1\76\25\37"+ - "\27\0\4\37\3\0\15\37\1\77\10\37\27\0\4\37"+ - "\3\0\17\37\1\100\6\37\27\0\4\37\3\0\14\37"+ - "\1\101\3\37\1\102\2\37\1\103\2\37\27\0\4\37"+ - "\3\0\17\37\1\104\6\37\27\0\4\37\3\0\12\37"+ - "\1\105\2\37\1\106\10\37\27\0\4\37\3\0\4\37"+ - "\1\107\21\37\27\0\4\37\3\0\12\37\1\110\11\37"+ - "\1\111\1\37\27\0\4\37\3\0\10\37\1\112\4\37"+ - "\1\113\1\37\1\114\6\37\27\0\4\37\3\0\12\37"+ - "\1\115\13\37\27\0\4\37\3\0\5\37\1\116\5\37"+ - "\1\117\1\120\3\37\1\121\5\37\27\0\4\37\3\0"+ - "\1\122\25\37\27\0\4\37\3\0\15\37\1\123\10\37"+ - "\27\0\4\37\3\0\17\37\1\124\6\37\27\0\4\37"+ - "\3\0\1\125\25\37\27\0\4\37\3\0\1\126\3\37"+ - "\1\127\21\37\27\0\4\37\3\0\17\37\1\130\6\37"+ - "\27\0\4\37\3\0\7\37\1\131\1\132\15\37\27\0"+ - "\4\37\3\0\10\37\1\133\15\37\6\65\1\0\25\65"+ - "\1\134\27\65\2\0\2\135\1\136\1\0\1\65\25\0"+ - "\1\65\15\0\1\65\2\0\1\65\1\0\1\65\33\0"+ - "\4\37\3\0\12\37\1\137\13\37\27\0\4\37\3\0"+ - "\14\37\1\140\11\37\27\0\4\37\3\0\22\37\1\141"+ - "\3\37\27\0\4\37\3\0\3\37\1\142\22\37\27\0"+ - "\4\37\3\0\20\37\1\143\5\37\27\0\4\37\3\0"+ - "\1\144\25\37\27\0\4\37\3\0\4\37\1\145\21\37"+ - "\27\0\4\37\3\0\1\146\25\37\27\0\4\37\3\0"+ - "\14\37\1\147\11\37\27\0\4\37\3\0\5\37\1\150"+ - "\4\37\1\151\13\37\27\0\4\37\3\0\10\37\1\152"+ - "\7\37\1\153\5\37\27\0\4\37\3\0\2\37\1\154"+ - "\23\37\27\0\4\37\3\0\14\37\1\155\11\37\27\0"+ - "\4\37\3\0\17\37\1\156\6\37\27\0\4\37\3\0"+ - "\15\37\1\157\10\37\27\0\4\37\3\0\15\37\1\160"+ - "\10\37\27\0\4\37\3\0\16\37\1\161\7\37\27\0"+ - "\4\37\3\0\13\37\1\162\12\37\27\0\4\37\3\0"+ - "\14\37\1\163\4\37\1\164\4\37\27\0\4\37\3\0"+ - "\20\37\1\165\5\37\27\0\4\37\3\0\10\37\1\166"+ - "\15\37\27\0\4\37\3\0\21\37\1\167\4\37\27\0"+ - "\4\37\3\0\25\37\1\170\27\0\4\37\3\0\10\37"+ - "\1\171\15\37\27\0\4\37\3\0\21\37\1\172\4\37"+ - "\27\0\4\37\3\0\4\37\1\173\21\37\6\0\1\65"+ - "\25\0\1\65\15\0\1\65\2\0\1\65\1\0\1\65"+ - "\6\0\1\135\110\0\4\37\3\0\20\37\1\174\5\37"+ - "\27\0\4\37\3\0\4\37\1\175\21\37\27\0\4\37"+ - "\3\0\4\37\1\176\21\37\27\0\4\37\3\0\4\37"+ - "\1\177\21\37\27\0\4\37\3\0\10\37\1\200\15\37"+ - "\27\0\4\37\3\0\1\201\25\37\27\0\4\37\3\0"+ - "\20\37\1\202\5\37\27\0\4\37\3\0\21\37\1\203"+ - "\4\37\27\0\4\37\3\0\5\37\1\204\20\37\27\0"+ - "\4\37\3\0\4\37\1\205\21\37\27\0\4\37\3\0"+ - "\4\37\1\206\21\37\27\0\4\37\3\0\1\207\25\37"+ - "\27\0\4\37\3\0\13\37\1\210\12\37\27\0\4\37"+ - "\3\0\1\37\1\211\24\37\27\0\4\37\3\0\15\37"+ - "\1\212\10\37\27\0\4\37\3\0\1\37\1\213\24\37"+ - "\27\0\4\37\3\0\12\37\1\214\13\37\27\0\4\37"+ - "\3\0\20\37\1\215\5\37\27\0\4\37\3\0\20\37"+ - "\1\216\5\37\27\0\4\37\3\0\22\37\1\217\3\37"+ - "\27\0\4\37\3\0\12\37\1\220\13\37\27\0\4\37"+ - "\3\0\7\37\1\221\16\37\27\0\4\37\3\0\12\37"+ - "\1\222\13\37\27\0\4\37\3\0\4\37\1\223\21\37"+ - "\27\0\4\37\3\0\17\37\1\224\6\37\27\0\4\37"+ - "\3\0\21\37\1\225\4\37\27\0\4\37\3\0\11\37"+ - "\1\226\14\37\27\0\4\37\3\0\20\37\1\227\5\37"+ - "\27\0\4\37\3\0\10\37\1\230\15\37\27\0\4\37"+ - "\3\0\16\37\1\231\7\37\27\0\4\37\3\0\12\37"+ - "\1\232\13\37\27\0\4\37\3\0\1\233\25\37\27\0"+ - "\4\37\3\0\17\37\1\234\6\37\27\0\4\37\3\0"+ - "\3\37\1\235\22\37\27\0\4\37\3\0\15\37\1\236"+ - "\10\37\27\0\4\37\3\0\4\37\1\237\21\37\27\0"+ - "\4\37\3\0\17\37\1\240\6\37\27\0\4\37\3\0"+ - "\4\37\1\241\21\37\27\0\4\37\3\0\3\37\1\242"+ - "\22\37\27\0\4\37\3\0\21\37\1\243\4\37\27\0"+ - "\4\37\3\0\14\37\1\244\11\37\27\0\4\37\3\0"+ - "\21\37\1\245\4\37\27\0\4\37\3\0\12\37\1\246"+ - "\13\37\27\0\4\37\3\0\12\37\1\247\13\37\27\0"+ - "\4\37\3\0\21\37\1\250\4\37\27\0\4\37\3\0"+ - "\1\251\25\37\27\0\4\37\3\0\2\37\1\252\23\37"+ - "\27\0\4\37\3\0\14\37\1\253\11\37\27\0\4\37"+ - "\3\0\22\37\1\254\3\37\27\0\4\37\3\0\25\37"+ - "\1\255\27\0\4\37\3\0\1\256\25\37\27\0\4\37"+ - "\3\0\4\37\1\257\21\37\27\0\4\37\3\0\12\37"+ - "\1\260\13\37"; - - private static int [] zzUnpackTrans() { - int [] result = new int[6136]; - int offset = 0; - offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackTrans(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - value--; - do result[j++] = value; while (--count > 0); - } - return j; - } - - - /** Error code for "Unknown internal scanner error". */ - private static final int ZZ_UNKNOWN_ERROR = 0; - /** Error code for "could not match input". */ - private static final int ZZ_NO_MATCH = 1; - /** Error code for "pushback value was too large". */ - private static final int ZZ_PUSHBACK_2BIG = 2; - - /** - * Error messages for {@link #ZZ_UNKNOWN_ERROR}, {@link #ZZ_NO_MATCH}, and - * {@link #ZZ_PUSHBACK_2BIG} respectively. - */ - private static final String ZZ_ERROR_MSG[] = { - "Unknown internal scanner error", - "Error: could not match input", - "Error: pushback value was too large" - }; - - /** - * ZZ_ATTRIBUTE[aState] contains the attributes of state {@code aState} - */ - private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute(); - - private static final String ZZ_ATTRIBUTE_PACKED_0 = - "\2\0\1\1\3\11\2\1\3\11\2\1\1\11\1\1"+ - "\6\11\1\1\1\11\1\1\1\11\1\1\1\11\7\1"+ - "\2\11\21\1\1\11\1\1\6\11\36\1\1\0\1\11"+ - "\123\1"; - - private static int [] zzUnpackAttribute() { - int [] result = new int[176]; - int offset = 0; - offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackAttribute(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - /** Input device. */ - private java.io.Reader zzReader; - - /** Current state of the DFA. */ - private int zzState; - - /** Current lexical state. */ - private int zzLexicalState = YYINITIAL; - - /** - * This buffer contains the current text to be matched and is the source of the {@link #yytext()} - * string. - */ - private char zzBuffer[] = new char[ZZ_BUFFERSIZE]; - - /** Text position at the last accepting state. */ - private int zzMarkedPos; - - /** Current text position in the buffer. */ - private int zzCurrentPos; - - /** Marks the beginning of the {@link #yytext()} string in the buffer. */ - private int zzStartRead; - - /** Marks the last character in the buffer, that has been read from input. */ - private int zzEndRead; - - /** - * Whether the scanner is at the end of file. - * @see #yyatEOF - */ - private boolean zzAtEOF; - - /** - * The number of occupied positions in {@link #zzBuffer} beyond {@link #zzEndRead}. - * - *

When a lead/high surrogate has been read from the input stream into the final - * {@link #zzBuffer} position, this will have a value of 1; otherwise, it will have a value of 0. - */ - private int zzFinalHighSurrogate = 0; - - /** Number of newlines encountered up to the start of the matched text. */ - private int yyline; - - /** Number of characters from the last newline up to the start of the matched text. */ - private int yycolumn; - - /** Number of characters up to the start of the matched text. */ - @SuppressWarnings("unused") - private long yychar; - - /** Whether the scanner is currently at the beginning of a line. */ - @SuppressWarnings("unused") - private boolean zzAtBOL = true; - - /** Whether the user-EOF-code has already been executed. */ - @SuppressWarnings("unused") - private boolean zzEOFDone; - - /* user code: */ - /* The code below includes some convenience methods to create tokens - * of a given type and optionally a value that the CUP parser can - * understand. Specifically, a lot of the logic below deals with - * embedded information about where in the source code a given token - * was recognized, so that the parser can report errors accurately. - * (It need not be modified for this project.) */ - - /** Producer of token-related values for the parser. */ - final ComplexSymbolFactory symbolFactory = new ComplexSymbolFactory(); - private int currIndent = 0; - private int currString = ""; - private ArrayList stack = new ArrayList(20); - /** Return a terminal symbol of syntactic category TYPE and no - * semantic value at the current source location. */ - private Symbol symbol(int type) { - return symbol(type); - } - - /** Return a terminal symbol of syntactic category TYPE and semantic - * value VALUE at the current source location. */ - private Symbol symbol(int type, Object value) { - return symbolFactory.newSymbol(ChocoPyTokens.terminalNames[type], type, - new ComplexSymbolFactory.Location(yyline + 1, yycolumn + 1), - new ComplexSymbolFactory.Location(yyline + 1,yycolumn + yylength()), - value); - } - private void push(int indent){ - stack.add(indent); - } - private int pop(){ - if(stack.isEmpty()) return 0; - return stack.remove(stack.size() - 1); - } - private int top(){ - if(stack.isEmpty) return 0; - return stack.get(stack.size() - 1); - } - - - /** - * Creates a new scanner - * - * @param in the java.io.Reader to read input from. - */ - public ChocoPyLexer(java.io.Reader in) { - this.zzReader = in; - } - - /** - * Translates raw input code points to DFA table row - */ - private static int zzCMap(int input) { - int offset = input & 255; - return offset == input ? ZZ_CMAP_BLOCKS[offset] : ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input >> 8] | offset]; - } - - /** - * Refills the input buffer. - * - * @return {@code false} iff there was new input. - * @exception java.io.IOException if any I/O-Error occurs - */ - private boolean zzRefill() throws java.io.IOException { - - /* first: make room (if you can) */ - if (zzStartRead > 0) { - zzEndRead += zzFinalHighSurrogate; - zzFinalHighSurrogate = 0; - System.arraycopy(zzBuffer, zzStartRead, - zzBuffer, 0, - zzEndRead - zzStartRead); - - /* translate stored positions */ - zzEndRead -= zzStartRead; - zzCurrentPos -= zzStartRead; - zzMarkedPos -= zzStartRead; - zzStartRead = 0; - } - - /* is the buffer big enough? */ - if (zzCurrentPos >= zzBuffer.length - zzFinalHighSurrogate) { - /* if not: blow it up */ - char newBuffer[] = new char[zzBuffer.length * 2]; - System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length); - zzBuffer = newBuffer; - zzEndRead += zzFinalHighSurrogate; - zzFinalHighSurrogate = 0; - } - - /* fill the buffer with new input */ - int requested = zzBuffer.length - zzEndRead; - int numRead = zzReader.read(zzBuffer, zzEndRead, requested); - - /* not supposed to occur according to specification of java.io.Reader */ - if (numRead == 0) { - throw new java.io.IOException( - "Reader returned 0 characters. See JFlex examples/zero-reader for a workaround."); - } - if (numRead > 0) { - zzEndRead += numRead; - if (Character.isHighSurrogate(zzBuffer[zzEndRead - 1])) { - if (numRead == requested) { // We requested too few chars to encode a full Unicode character - --zzEndRead; - zzFinalHighSurrogate = 1; - } else { // There is room in the buffer for at least one more char - int c = zzReader.read(); // Expecting to read a paired low surrogate char - if (c == -1) { - return true; - } else { - zzBuffer[zzEndRead++] = (char)c; - } - } - } - /* potentially more input available */ - return false; - } - - /* numRead < 0 ==> end of stream */ - return true; - } - - - /** - * Closes the input reader. - * - * @throws java.io.IOException if the reader could not be closed. - */ - public final void yyclose() throws java.io.IOException { - zzAtEOF = true; // indicate end of file - zzEndRead = zzStartRead; // invalidate buffer - - if (zzReader != null) { - zzReader.close(); - } - } - - - /** - * Resets the scanner to read from a new input stream. - * - *

Does not close the old reader. - * - *

All internal variables are reset, the old input stream cannot be reused (internal - * buffer is discarded and lost). Lexical state is set to {@code ZZ_INITIAL}. - * - *

Internal scan buffer is resized down to its initial length, if it has grown. - * - * @param reader The new input stream. - */ - public final void yyreset(java.io.Reader reader) { - zzReader = reader; - zzEOFDone = false; - yyResetPosition(); - zzLexicalState = YYINITIAL; - if (zzBuffer.length > ZZ_BUFFERSIZE) { - zzBuffer = new char[ZZ_BUFFERSIZE]; - } - } - - /** - * Resets the input position. - */ - private final void yyResetPosition() { - zzAtBOL = true; - zzAtEOF = false; - zzCurrentPos = 0; - zzMarkedPos = 0; - zzStartRead = 0; - zzEndRead = 0; - zzFinalHighSurrogate = 0; - yyline = 0; - yycolumn = 0; - yychar = 0L; - } - - - /** - * Returns whether the scanner has reached the end of the reader it reads from. - * - * @return whether the scanner has reached EOF. - */ - public final boolean yyatEOF() { - return zzAtEOF; - } - - - /** - * Returns the current lexical state. - * - * @return the current lexical state. - */ - public final int yystate() { - return zzLexicalState; - } - - - /** - * Enters a new lexical state. - * - * @param newState the new lexical state - */ - public final void yybegin(int newState) { - zzLexicalState = newState; - } - - - /** - * Returns the text matched by the current regular expression. - * - * @return the matched text. - */ - public final String yytext() { - return new String(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); - } - - - /** - * Returns the character at the given position from the matched text. - * - *

It is equivalent to {@code yytext().charAt(pos)}, but faster. - * - * @param position the position of the character to fetch. A value from 0 to {@code yylength()-1}. - * - * @return the character at {@code position}. - */ - public final char yycharat(int position) { - return zzBuffer[zzStartRead + position]; - } - - - /** - * How many characters were matched. - * - * @return the length of the matched text region. - */ - public final int yylength() { - return zzMarkedPos-zzStartRead; - } - - - /** - * Reports an error that occurred while scanning. - * - *

In a well-formed scanner (no or only correct usage of {@code yypushback(int)} and a - * match-all fallback rule) this method will only be called with things that - * "Can't Possibly Happen". - * - *

If this method is called, something is seriously wrong (e.g. a JFlex bug producing a faulty - * scanner etc.). - * - *

Usual syntax/scanner level error handling should be done in error fallback rules. - * - * @param errorCode the code of the error message to display. - */ - private static void zzScanError(int errorCode) { - String message; - try { - message = ZZ_ERROR_MSG[errorCode]; - } catch (ArrayIndexOutOfBoundsException e) { - message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; - } - - throw new Error(message); - } - - - /** - * Pushes the specified amount of characters back into the input stream. - * - *

They will be read again by then next call of the scanning method. - * - * @param number the number of characters to be read again. This number must not be greater than - * {@link #yylength()}. - */ - public void yypushback(int number) { - if ( number > yylength() ) - zzScanError(ZZ_PUSHBACK_2BIG); - - zzMarkedPos -= number; - } - - - - - /** - * Resumes scanning until the next regular expression is matched, the end of input is encountered - * or an I/O-Error occurs. - * - * @return the next token. - * @exception java.io.IOException if any I/O-Error occurs. - */ - @Override public java_cup.runtime.Symbol next_token() throws java.io.IOException { - int zzInput; - int zzAction; - - // cached fields: - int zzCurrentPosL; - int zzMarkedPosL; - int zzEndReadL = zzEndRead; - char[] zzBufferL = zzBuffer; - - int [] zzTransL = ZZ_TRANS; - int [] zzRowMapL = ZZ_ROWMAP; - int [] zzAttrL = ZZ_ATTRIBUTE; - - while (true) { - zzMarkedPosL = zzMarkedPos; - - boolean zzR = false; - int zzCh; - int zzCharCount; - for (zzCurrentPosL = zzStartRead ; - zzCurrentPosL < zzMarkedPosL ; - zzCurrentPosL += zzCharCount ) { - zzCh = Character.codePointAt(zzBufferL, zzCurrentPosL, zzMarkedPosL); - zzCharCount = Character.charCount(zzCh); - switch (zzCh) { - case '\u000B': // fall through - case '\u000C': // fall through - case '\u0085': // fall through - case '\u2028': // fall through - case '\u2029': - yyline++; - yycolumn = 0; - zzR = false; - break; - case '\r': - yyline++; - yycolumn = 0; - zzR = true; - break; - case '\n': - if (zzR) - zzR = false; - else { - yyline++; - yycolumn = 0; - } - break; - default: - zzR = false; - yycolumn += zzCharCount; - } - } - - if (zzR) { - // peek one character ahead if it is - // (if we have counted one line too much) - boolean zzPeek; - if (zzMarkedPosL < zzEndReadL) - zzPeek = zzBufferL[zzMarkedPosL] == '\n'; - else if (zzAtEOF) - zzPeek = false; - else { - boolean eof = zzRefill(); - zzEndReadL = zzEndRead; - zzMarkedPosL = zzMarkedPos; - zzBufferL = zzBuffer; - if (eof) - zzPeek = false; - else - zzPeek = zzBufferL[zzMarkedPosL] == '\n'; - } - if (zzPeek) yyline--; - } - zzAction = -1; - - zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL; - - zzState = ZZ_LEXSTATE[zzLexicalState]; - - // set up zzAction for empty match case: - int zzAttributes = zzAttrL[zzState]; - if ( (zzAttributes & 1) == 1 ) { - zzAction = zzState; - } - - - zzForAction: { - while (true) { - - if (zzCurrentPosL < zzEndReadL) { - zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL); - zzCurrentPosL += Character.charCount(zzInput); - } - else if (zzAtEOF) { - zzInput = YYEOF; - break zzForAction; - } - else { - // store back cached positions - zzCurrentPos = zzCurrentPosL; - zzMarkedPos = zzMarkedPosL; - boolean eof = zzRefill(); - // get translated positions and possibly new buffer - zzCurrentPosL = zzCurrentPos; - zzMarkedPosL = zzMarkedPos; - zzBufferL = zzBuffer; - zzEndReadL = zzEndRead; - if (eof) { - zzInput = YYEOF; - break zzForAction; - } - else { - zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL); - zzCurrentPosL += Character.charCount(zzInput); - } - } - int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMap(zzInput) ]; - if (zzNext == -1) break zzForAction; - zzState = zzNext; - - zzAttributes = zzAttrL[zzState]; - if ( (zzAttributes & 1) == 1 ) { - zzAction = zzState; - zzMarkedPosL = zzCurrentPosL; - if ( (zzAttributes & 8) == 8 ) break zzForAction; - } - - } - } - - // store back cached position - zzMarkedPos = zzMarkedPosL; - - if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { - zzAtEOF = true; - { - return symbol(ChocoPyTokens.EOF); - } - } - else { - switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) { - case 1: - { currString+=yytext(); - } - // fall through - case 68: break; - case 2: - { yypushback(1); - if(top() > currIndent) - { - pop(); - return symbol(ChocoPyTokens.DEDENT); - } - yystart(AFTER); - if(top()< currIndent) - { - push(currIndent); - return symbol(ChocoPyTokens.INDENT); - } - } - // fall through - case 69: break; - case 3: - { String space = yytext(); - if(space == "\t") - currIndent += 8; - else - currIndent ++; - } - // fall through - case 70: break; - case 4: - { currIndent = 0; - } - // fall through - case 71: break; - case 5: - { /* ignored */ - } - // fall through - case 72: break; - case 6: - { return symbol(ChocoPyTokens.UNRECOGNIZED); - } - // fall through - case 73: break; - case 7: - { /* ignore */ - } - // fall through - case 74: break; - case 8: - { return symbol(ChocoPyTokens.NEWLINE); yybegin(YYINITIAL); currIndent = 0; - } - // fall through - case 75: break; - case 9: - { yystart(STR); currString = ""; - } - // fall through - case 76: break; - case 10: - { return symbol(ChocoPyTokens.MOD); - } - // fall through - case 77: break; - case 11: - { return symbol(ChocoPyTokens.LPAR); - } - // fall through - case 78: break; - case 12: - { return symbol(ChocoPyTokens.RPAR); - } - // fall through - case 79: break; - case 13: - { return symbol(ChocoPyTokens.MUL); - } - // fall through - case 80: break; - case 14: - { return symbol(ChocoPyTokens.PLUS); - } - // fall through - case 81: break; - case 15: - { return symbol(ChocoPyTokens.COMMA); - } - // fall through - case 82: break; - case 16: - { return symbol(ChocoPyTokens.MINUS); - } - // fall through - case 83: break; - case 17: - { return symbol(ChocoPyTokens.DOT); - } - // fall through - case 84: break; - case 18: - { return symbol(ChocoPyTokens.DIV); - } - // fall through - case 85: break; - case 19: - { return symbol(ChocoPyTokens.NUMBER, - Integer.parseInt(yytext())); - } - // fall through - case 86: break; - case 20: - { return symbol(ChocoPyTokens.COLON); - } - // fall through - case 87: break; - case 21: - { return symbol(ChocoPyTokens.LT); - } - // fall through - case 88: break; - case 22: - { return symbol(ChocoPyTokens.ASSIGN); - } - // fall through - case 89: break; - case 23: - { return symbol(ChocoPyTokens.GT); - } - // fall through - case 90: break; - case 24: - { return symbol(ChocoPyTokens.NAMES, yytext()); - } - // fall through - case 91: break; - case 25: - { return symbol(ChocoPyTokens.LBR); - } - // fall through - case 92: break; - case 26: - { return symbol(ChocoPyTokens.RBR); - } - // fall through - case 93: break; - case 27: - { yybegin(AFTER); return symbol(ChocoPyTokens.STRING, currString); - } - // fall through - case 94: break; - case 28: - { return symbol(ChocoPyTokens.NEQ); - } - // fall through - case 95: break; - case 29: - { return symbol(ChocoPyTokens.ARROW); - } - // fall through - case 96: break; - case 30: - { return symbol(ChocoPyTokens.LEQ); - } - // fall through - case 97: break; - case 31: - { return symbol(ChocoPyTokens.EQUAL); - } - // fall through - case 98: break; - case 32: - { return symbol(ChocoPyTokens.GEQ); - } - // fall through - case 99: break; - case 33: - { return symbol(ChocoPyTokens.AS); - } - // fall through - case 100: break; - case 34: - { return symbol(ChocoPyTokens.IF); - } - // fall through - case 101: break; - case 35: - { return symbol(ChocoPyTokens.IN); - } - // fall through - case 102: break; - case 36: - { return symbol(ChocoPyTokens.IS); - } - // fall through - case 103: break; - case 37: - { return symbol(ChocoPyTokens.OR); - } - // fall through - case 104: break; - case 38: - // lookahead expression with fixed base length - zzMarkedPos = Character.offsetByCodePoints - (zzBufferL, zzStartRead, zzEndRead - zzStartRead, zzStartRead, 1); - { /*'\' at the end of line, do nothing.*/ - } - // fall through - case 105: break; - case 39: - { return symbol(ChocoPyTokens.AND); - } - // fall through - case 106: break; - case 40: - { return symbol(ChocoPyTokens.DEF); - } - // fall through - case 107: break; - case 41: - { return symbol(ChocoPyTokens.DEL); - } - // fall through - case 108: break; - case 42: - { return symbol(ChocoPyTokens.FOR); - } - // fall through - case 109: break; - case 43: - { return symbol(ChocoPyTokens.NOT); - } - // fall through - case 110: break; - case 44: - { return symbol(ChocoPyTokens.TRY); - } - // fall through - case 111: break; - case 45: - { return symbol(ChocoPyTokens.NONE); - } - // fall through - case 112: break; - case 46: - { return symbol(ChocoPyTokens.BOOL, true); - } - // fall through - case 113: break; - case 47: - { return symbol(ChocoPyTokens.ELIF); - } - // fall through - case 114: break; - case 48: - { return symbol(ChocoPyTokens.ELSE); - } - // fall through - case 115: break; - case 49: - { return symbol(ChocoPyTokens.FROM); - } - // fall through - case 116: break; - case 50: - { return symbol(ChocoPyTokens.PASS); - } - // fall through - case 117: break; - case 51: - { return symbol(ChocoPyTokens.WITH); - } - // fall through - case 118: break; - case 52: - { return symbol(ChocoPyTokens.BOOL, false); - } - // fall through - case 119: break; - case 53: - { return symbol(ChocoPyTokens.AWAIT); - } - // fall through - case 120: break; - case 54: - { return symbol(ChocoPyTokens.BREAK); - } - // fall through - case 121: break; - case 55: - { return symbol(ChocoPyTokens.CLASS); - } - // fall through - case 122: break; - case 56: - { return symbol(ChocoPyTokens.RAISE); - } - // fall through - case 123: break; - case 57: - { return symbol(ChocoPyTokens.WHILE); - } - // fall through - case 124: break; - case 58: - { return symbol(ChocoPyTokens.YIELD); - } - // fall through - case 125: break; - case 59: - { return symbol(ChocoPyTokens.ASSERT); - } - // fall through - case 126: break; - case 60: - { return symbol(ChocoPyTokens.EXCEPT); - } - // fall through - case 127: break; - case 61: - { return symbol(ChocoPyTokens.GLOBAL); - } - // fall through - case 128: break; - case 62: - { return symbol(ChocoPyTokens.IMPORT); - } - // fall through - case 129: break; - case 63: - { return symbol(ChocoPyTokens.LAMBDA); - } - // fall through - case 130: break; - case 64: - { return symbol(ChocoPyTokens.RETURN); - } - // fall through - case 131: break; - case 65: - { return symbol(ChocoPyTokens.FINALLY); - } - // fall through - case 132: break; - case 66: - { return symbol(ChocoPyTokens.CONTINUE); - } - // fall through - case 133: break; - case 67: - { return symbol(ChocoPyTokens.NONLOCAL); - } - // fall through - case 134: break; - default: - zzScanError(ZZ_NO_MATCH); - } - } - } - } - - /** - * Converts an int token code into the name of the - * token by reflection on the cup symbol class/interface ChocoPyTokens - */ - private static String getTokenName(int token) { - try { - java.lang.reflect.Field [] classFields = ChocoPyTokens.class.getFields(); - for (int i = 0; i < classFields.length; i++) { - if (classFields[i].getInt(null) == token) { - return classFields[i].getName(); - } - } - } catch (Exception e) { - e.printStackTrace(System.err); - } - - return "UNKNOWN TOKEN"; - } - - /** - * Same as next_token but also prints the token to standard out - * for debugging. - */ - public java_cup.runtime.Symbol debug_next_token() throws java.io.IOException { - java_cup.runtime.Symbol s = next_token(); - System.out.println( "line:" + (yyline+1) + " col:" + (yycolumn+1) + " --"+ yytext() + "--" + getTokenName(s.sym) + "--"); - return s; - } - - /** - * Runs the scanner on input files. - * - * This main method is the debugging routine for the scanner. - * It prints debugging information about each returned token to - * System.out until the end of file is reached, or an error occured. - * - * @param argv the command line, contains the filenames to run - * the scanner on. - */ - public static void main(String[] argv) { - if (argv.length == 0) { - System.out.println("Usage : java ChocoPyLexer [ --encoding ] "); - } - else { - int firstFilePos = 0; - String encodingName = "UTF-8"; - if (argv[0].equals("--encoding")) { - firstFilePos = 2; - encodingName = argv[1]; - try { - // Side-effect: is encodingName valid? - java.nio.charset.Charset.forName(encodingName); - } catch (Exception e) { - System.out.println("Invalid encoding '" + encodingName + "'"); - return; - } - } - for (int i = firstFilePos; i < argv.length; i++) { - ChocoPyLexer scanner = null; - try { - java.io.FileInputStream stream = new java.io.FileInputStream(argv[i]); - java.io.Reader reader = new java.io.InputStreamReader(stream, encodingName); - scanner = new ChocoPyLexer(reader); - while ( !scanner.zzAtEOF ) scanner.debug_next_token(); - } - catch (java.io.FileNotFoundException e) { - System.out.println("File not found : \""+argv[i]+"\""); - } - catch (java.io.IOException e) { - System.out.println("IO error scanning file \""+argv[i]+"\""); - System.out.println(e); - } - catch (Exception e) { - System.out.println("Unexpected exception:"); - e.printStackTrace(); - } - } - } - } - - -}