ChocoPy/src/main/jflex/chocopy/pa1/ChocoPy.jflex

package chocopy.pa1;
import java_cup.runtime.*;
import java.util.ArrayList;
%%

/*** Do not change the flags below unless you know what you are doing. ***/

%unicode
%line
%column
%states AFTER, STR
%class ChocoPyLexer
%public

%cupsym ChocoPyTokens
%cup
%cupdebug

%eofclose false

/*** Do not change the flags above unless you know what you are doing. ***/

/* The following code section is copied verbatim to the
 * generated lexer class. */
%{
    /* The code below includes some convenience methods to create tokens
     * of a given type and optionally a value that the CUP parser can
     * understand. Specifically, a lot of the logic below deals with
     * embedded information about where in the source code a given token
     * was recognized, so that the parser can report errors accurately.
     * (It need not be modified for this project.) */

    /** Producer of token-related values for the parser. */
    final ComplexSymbolFactory symbolFactory = new ComplexSymbolFactory();
    private int currIndent = 0;
    private int currString = "";
    private ArrayList<Integer> stack = new ArrayList<Integer>(20);
    /** Return a terminal symbol of syntactic category TYPE and no
     *  semantic value at the current source location. */
    private Symbol symbol(int type) {
        return symbol(type);
    }

    /** Return a terminal symbol of syntactic category TYPE and semantic
     *  value VALUE at the current source location. */
    private Symbol symbol(int type, Object value) {
        return symbolFactory.newSymbol(ChocoPyTokens.terminalNames[type], type,
            new ComplexSymbolFactory.Location(yyline + 1, yycolumn + 1),
            new ComplexSymbolFactory.Location(yyline + 1,yycolumn + yylength()),
            value);
    }
    private void push(int indent){
        stack.add(indent);
    }
    private int pop(){
        if(stack.isEmpty()) return 0;
        return stack.remove(stack.size() - 1);
    }
    private int top(){
        if(stack.isEmpty) return 0;
        return stack.get(stack.size() - 1);
    }
%}

/* Macros (regexes used in rules below) */

WhiteSpace = [ \t]
LineBreak  = \r|\n|\r\n

IntegerLiteral = 0|[1-9][0-9]*
StringLiteral = ([^\"\\]|(\\\")|(\\t)|(\\r)|(\\n)|(\\\\))*
Names = (_|[a-z]|[A-Z])(_|[a-z]|[A-Z])*
Comments = #[^\r\n]*
%%

<YYINITIAL>{
  {WhiteSpace}                
  { 
      String space = yytext();
      if(space == "\t")
        currIndent += 8;
      else 
        currIndent ++;
  }
  {LineBreak}
  {
      currIndent = 0;
  }
  {Comments}           { /* ignored */ }
  [^ \t\r\n#]               
  {
      yypushback(1);
      if(top() > currIndent)
      {    
          pop();
          return symbol(ChocoPyTokens.DEDENT);
      }
      yystart(AFTER);
      if(top()< currIndent)
      {
          push(currIndent);
          return symbol(ChocoPyTokens.INDENT);
      }
  }
}
<AFTER> {

  /* Delimiters. */
  {LineBreak}                 { return symbol(ChocoPyTokens.NEWLINE); yybegin(YYINITIAL); currIndent = 0;}
  ":"                         { return symbol(ChocoPyTokens.COLON); }
  ","                         { return symbol(ChocoPyTokens.COMMA); }

  /* Literals. */
  {IntegerLiteral}            { return symbol(ChocoPyTokens.NUMBER,
                                                 Integer.parseInt(yytext())); }
//  {StringLiteral}             { return symbol(ChocoPyTokens.STRING, yytext());}
  "\""                           {yystart(STR); currString = "";} 
  "False"                        { return symbol(ChocoPyTokens.BOOL, false); }
  "True"                         { return symbol(ChocoPyTokens.BOOL, true); }
  "None"                         { return symbol(ChocoPyTokens.NONE); }

  /*Keywords*/
  "if"                          {return symbol(ChocoPyTokens.IF);}
  "else"                          {return symbol(ChocoPyTokens.ELSE);}
  "elif"                          {return symbol(ChocoPyTokens.ELIF);}
  "while"                          {return symbol(ChocoPyTokens.WHILE);}
  "class"                          {return symbol(ChocoPyTokens.CLASS);}
  "def"                          {return symbol(ChocoPyTokens.DEF);}
  "lambda"                          {return symbol(ChocoPyTokens.LAMBDA);}
  "as"                           { return symbol(ChocoPyTokens.AS); }
  "for"                          { return symbol(ChocoPyTokens.FOR); }
  "global"                       { return symbol(ChocoPyTokens.GLOBAL); }
  "in"                           { return symbol(ChocoPyTokens.IN); }
  "nonlocal"                     { return symbol(ChocoPyTokens.NONLOCAL); }
  "pass"                         { return symbol(ChocoPyTokens.PASS); }
  "return"                       { return symbol(ChocoPyTokens.RETURN); }
  "assert"                       { return symbol(ChocoPyTokens.ASSERT); }
  "await"                        { return symbol(ChocoPyTokens.AWAIT); }
  "break"                        { return symbol(ChocoPyTokens.BREAK); }
  "continue"                     { return symbol(ChocoPyTokens.CONTINUE); }
  "del"                          { return symbol(ChocoPyTokens.DEL); }
  "except"                       { return symbol(ChocoPyTokens.EXCEPT); }
  "finally"                      { return symbol(ChocoPyTokens.FINALLY); }
  "from"                         { return symbol(ChocoPyTokens.FROM); }
  "import"                       { return symbol(ChocoPyTokens.IMPORT); }
  "raise"                        { return symbol(ChocoPyTokens.RAISE); }
  "try"                          { return symbol(ChocoPyTokens.TRY); }
  "with"                         { return symbol(ChocoPyTokens.WITH); }
  "yield"                        { return symbol(ChocoPyTokens.YIELD); }


  /* Operators. */
  "+"                         { return symbol(ChocoPyTokens.PLUS); }
  "-"                         { return symbol(ChocoPyTokens.MINUS); }
  "*"                         { return symbol(ChocoPyTokens.MUL); }
  "//"                         { return symbol(ChocoPyTokens.DIV); }  
  "/"                         { return symbol(ChocoPyTokens.DIV); }  
  "%"                         { return symbol(ChocoPyTokens.MOD); }  
  ">"                         { return symbol(ChocoPyTokens.GT); }
  "<"                         { return symbol(ChocoPyTokens.LT); }
  "=="                         { return symbol(ChocoPyTokens.EQUAL); }
  "!="                         { return symbol(ChocoPyTokens.NEQ); }
  ">="                         { return symbol(ChocoPyTokens.GEQ); }
  "<="                         { return symbol(ChocoPyTokens.LEQ); }
  "="                         { return symbol(ChocoPyTokens.ASSIGN); }
  "and"                         { return symbol(ChocoPyTokens.AND); }
  "or"                         { return symbol(ChocoPyTokens.OR); }
  "not"                         { return symbol(ChocoPyTokens.NOT); }
  "."                         { return symbol(ChocoPyTokens.DOT); }
  "("                         { return symbol(ChocoPyTokens.LPAR); }
  ")"                         { return symbol(ChocoPyTokens.RPAR); }
  "["                         { return symbol(ChocoPyTokens.LBR); }
  "]"                         { return symbol(ChocoPyTokens.RBR); }
  "->"                           { return symbol(ChocoPyTokens.ARROW); }
  "is"                           { return symbol(ChocoPyTokens.IS); }
  
 
  /*Identifiers*/
  {Names}                     {return symbol(ChocoPyTokens.NAMES, yytext());}
  /* Whitespace. */
  {WhiteSpace}                { /* ignore */ }
  /* Comment. */
  {Comments}                  { /* ignore */ }
}
<STR>{
    {StringLiteral}           {currString+=yytext();}
    "\""                      {yybegin(AFTER); return symbol(ChocoPyTokens.STRING, currString);}
}
<<EOF>>                       { return symbol(ChocoPyTokens.EOF); }

/* Error fallback. */
[^]                           { return symbol(ChocoPyTokens.UNRECOGNIZED); }