diff --git a/knownToBeUnsupported.md b/knownToBeUnsupported.md new file mode 100644 index 0000000..786c25d --- /dev/null +++ b/knownToBeUnsupported.md @@ -0,0 +1,32 @@ +# Currently known to be unsupported + +## Tuples without parentheses +The usage of tuples that are not parenthesized is not supported as these causes issues with antlr. (See https://www.geeksforgeeks.org/python/when-are-parentheses-required-around-a-tuple-in-python/ for unparenthesized tuples) + We define the rules: + + ```TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ;``` + ```SimpleInit implements VariableInit = Expression ;``` + and ExpressionBasis.mc4 ```LiteralExpression implements Expression <340> = Literal;``` + + Thereby without the parentheses we would cause left recursion which is also reachable by ``LiteralStatement implements ClassStatement = Literal STATEMENT_END;```. + +## Unicode names + Further for now only names with latin letters are permitted, in python another chars are allowed as well, see https://docs.python.org/3/reference/lexical_analysis.html#identifiers. + Prototyping with unicode names have passed the parser tests, further testing needs to be done. + +``` @Override + token Name = + ( UnicodeChar | '_' | '$' ) + ( UnicodeChar | '_' | '0'..'9' | '$' )*; + // Latin,Greek,Coptic,Cyrillic,Armenian + fragment token UnicodeChar = 'a'..'z' + |'A'..'Z' + |'\u00C0'..'\u00D6' + |'\u00D8'..'\u00F6' + |'\u00F8'..'\u02AF' + |'\u0370'..'\u0373' + |'\u0376' | '\u0377' | '\u037F' | '\u0386' + |'\u0386'..'\u03E1' + |'\u03E2'..'\u0481' + |'\u048A'..'\u0588'; +``` diff --git a/src/main/grammars/de/monticore/MultilineString.mc4 b/src/main/grammars/de/monticore/MultilineString.mc4 index 04b5981..b287110 100644 --- a/src/main/grammars/de/monticore/MultilineString.mc4 +++ b/src/main/grammars/de/monticore/MultilineString.mc4 @@ -5,5 +5,6 @@ package de.monticore; // This can be fixed by adding another token, which must be defined before the String token. // Thus, this grammar must be used before MCCommonLiterals, which most langauges use. component grammar MultilineString { - token DoubleQuoteMultilineStringDelimiter = '"' '"' '"'; + token DoubleQuoteMultilineFStringDelimiter = ('f'|'F') '"' '"' '"'; + token DoubleQuoteMultilineStringDelimiter = '"' '"' '"'; } \ No newline at end of file diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 0f4e87d..af32c08 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -1,13 +1,7 @@ /* (c) https://github.com/MontiCore/monticore */ package de.monticore; - /* This is a MontiCore alpha grammar. Adaptions are possible */ - -import de.monticore.MultilineString; -import de.monticore.expressions.*; -import de.monticore.literals.MCCommonLiterals; -import de.monticore.symbols.OOSymbols; - +import de.monticore.PythonBasis; /** * The Python language defines an almost complete subset of the Python programming language. * The goal is to analyze existing Python (research) software; therefore, the grammar and CoCos are somewhat simplified. @@ -15,104 +9,10 @@ import de.monticore.symbols.OOSymbols; * e.g., accepting code that is not strictly valid Python. * The focus is instead on building an abstract syntax that is easy to analyze within the MontiCore toolchain. */ -grammar Python extends MultilineString, // Must be first to avoid conflicts with String token! - CommonExpressions, - AssignmentExpressions, - MCCommonLiterals, - OOSymbols { +grammar Python extends PythonBasis { PythonScript = Statement*; - /*====================================== Tokens ======================================*/ - @Override - token WS = (' ' | '\t' | '\r' | '\n' ) : -> channel(HIDDEN); - @Override - token SL_COMMENT = "#" (~('\n' | '\r' ))* : -> channel(HIDDEN); - token ByteOrderMark = '\uFEFF' : -> skip; - - /** - * The following utf-8 symbols are used to parse the Python files without having to add whitespace to the Grammar. - * In a preprocessing step code blocks are denoted with \u204f = ⦃ and \u2984 = ⦄, and lines are ended with \u204f = ⁏ - */ - token BLOCK_START = '\u2983'; - token BLOCK_END = '\u2984'; - token STATEMENT_END = ';'? '\u204f' | ';' '\u204f'?; - - // Will be filtered out by the WhitespacePreprocessingTokenSource - // Used break a line without finishing the statement - token CONTINUE_LINE_TOKEN = '\\' '\r'? '\n'; - - // === string tokens for python === - // Often (mis)used as a multiline comment but can also be used as a string literal, thus we can not skip it - token MultiLineStringToken = ((("\'\'\'") .*? ("\'\'\'")) | ((DoubleQuoteMultilineStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); - - @Override - token String = '"' (StringDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; - - token StringPython - = '\'' (StringSQCharactersPython)? '\'' - : {setText(getText().substring(1, getText().length() - 1));}; - - fragment token StringSQCharactersPython - = (StringSQCharacterPython)+; - fragment token StringDQCharactersPython - = (StringDQCharacterPython)+; - - fragment token StringSQCharacterPython - = ~ ('\''| '\\' | '\n') | PythonEscapeSequence; - fragment token StringDQCharacterPython - = ~ ('"'| '\\' | '\n') | PythonEscapeSequence; - - fragment token PythonEscapeSequence - = '\\' .; - - // === number tokens for python === - - token FloatWithExponent = (DigitsPart | PyFloat) ('e'|'E') ('+' | '-')? DigitsPart; - FloatWithExponentLiteral implements NumericLiteral <100> = FloatWithExponent; - - token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.'; - PyFloatLiteral implements NumericLiteral <200> = PyFloat; - - // PEP 515 - token DigitsPart = Digit ('_'? Digit)*; - - token HexNumberToken = '0' 'x' ('0'..'9' | 'a'..'f' | 'A'..'F')+; - // PEP 515: Underscores in Numeric Literals - @Override - token Digits - = Digit ('_'? Digit)*; // technically the first digit must be nonzero except for 0(_0)* - - /*====================================== Literals ======================================*/ - - HexNumberLiteral implements NumericLiteral <100> = HexNumberToken; - - ArrayLiteral implements Literal = "[" (VariableInit || ",")* ","? "]" ; - TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ; - DictLiteral implements Literal = "{" (DictEntry || ",")* ","? "}" ; - SetLiteral implements Literal = "{" (Expression || ",")* ","? "}" ; - - DictEntry = key:VariableInit ":" value:VariableInit | SpreadMappingExpression; - - StringLiteralPython implements Literal, SignedLiteral = - ( - StringModifier? - (source:StringPython | source:String) - ); - - StringsLiteralPython implements Literal <200> = (StringLiteralPython | StringLiteral | MultiLineStringLiteral)+; - - StringModifier = /*{cmpTokenRegEx(1, "(f|r|b|u|F|R|B|U)+")}?*/ type:Name; - MultiLineStringLiteral implements Literal = StringModifier? MultiLineStringToken; - - // boolean literals for python - BooleanLiteralPython implements Literal, SignedLiteral = - source:["True" | "False"]; - - // https://docs.python.org/dev/library/constants.html#Ellipsis - EllipsisLiteral implements Literal = "..."; - splittoken "..."; - /*====================================== Statements ======================================*/ interface Statement; @@ -120,19 +20,8 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit scope StatementBlock = (BLOCK_START StatementBlockBody BLOCK_END) | Statement; StatementBlockBody = Statement+; - LiteralStatement implements ClassStatement = Literal STATEMENT_END; - - PassStatement implements Statement, ClassStatement = "pass" STATEMENT_END; - - ElseStatementPart = "else" ":" StatementBlock; - - PyQualifiedName = (Name || ".")+; - astrule PyQualifiedName = method public String joined(){ - return String.join(".", getNameList()); - }; - // import statement - ImportStatement implements Statement = + ImportStatement implements Statement,ClassStatement = ("from" {!next("import")}? (leadingDots:"."*) (module:PyQualifiedName)?)? "import" ( @@ -145,36 +34,31 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ModuleWithOptionalAlias = name:PyQualifiedName ("as" alias:Name)?; + //Conditional statements + IfStatement implements Statement,ClassStatement = "if" condition:Expression ":" thenStatement:StatementBlock + ("elif" elifCondition:Expression ":" elifStatement:StatementBlock )* + ElseStatementPart?; + ElseStatementPart = "else" ":" StatementBlock; + ConditionalExecutionStatement implements Statement ,ClassStatement= "if" condition: Expression ":" Expression; - // if-else statement - IfStatement implements Statement = "if" condition:Expression ":" thenStatement:StatementBlock - ("elif" elifCondition:Expression ":" elifStatement:StatementBlock )* - ElseStatementPart?; - - // assert statement - AssertStatement implements Statement = "assert" condition:Expression ("," errorMessage:Expression)? STATEMENT_END; - - // for statement - scope ForStatement implements Statement = async:"async"? "for" ForControl ":" StatementBlock - ElseStatementPart? ; - - ForControl = ForDecomposition "in" ForIterable; - interface ForDecomposition; - ForVariable implements Variable, ForDecomposition = Name; - ForDecompositionComma implements ForDecomposition = ForDecomposition "," (ForDecomposition ","?)?; - ForDecompositionParenthesis implements ForDecomposition = "(" ForDecomposition ")"; + //assert statement + AssertStatement implements Statement,ClassStatement = "assert" condition:Expression ("," errorMessage:Expression)? STATEMENT_END; - ForIterable = Expression; + //for statement + scope ForStatement implements Statement,ClassStatement = async:"async"? "for" ForControl ":" StatementBlock + ElseStatementPart? ; - // while statement - WhileStatement implements Statement = "while" condition:Expression ":" StatementBlock - ElseStatementPart? ; + //while statement + WhileStatement implements Statement,ClassStatement = "while" condition:Expression ":" StatementBlock + ElseStatementPart? ; + //Control-flow statements + PassStatement implements Statement, ClassStatement = "pass" STATEMENT_END; BreakStatement implements Statement = "break" STATEMENT_END; ContinueStatement implements Statement = "continue" STATEMENT_END; // try-except-finally statement - TryExceptStatement implements Statement = "try" ":" tryStatement:StatementBlock + TryExceptStatement implements Statement,ClassStatement = "try" ":" tryStatement:StatementBlock ( ( ExceptStatement+ @@ -184,17 +68,25 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit | FinallyStatement ) ; - - ExceptStatement = "except" (PyQualifiedName? | "(" (PyQualifiedName || ",")+ ")") ("as" alias:Name)? ":" StatementBlock; + scope ExceptStatement = "except" ExceptPattern? ":" ExceptStatementBlock; + ExceptStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; FinallyStatement = "finally" ":" finallyStatement:StatementBlock; - // with open file statement - scope WithStatement implements Statement = async:"async"? "with" (WithStatementContents || ",")+ ":" - StatementBlock ; + //PEP758 + interface ExceptPattern; + AliasedSingleExpressionPattern implements ExceptPattern = Expression ("as" Alias)?; + ExpressionListing implements ExceptPattern = (Expression || ",")+; + ParenthesisedExpressionListing implements ExceptPattern = "(" (Expression || ",")+ ")" ("as" Alias)?; + StarredExpressionListing implements ExceptPattern = "*"(Expression || ",")+; + StarredParenthesisedExpressionListing implements ExceptPattern = "*""(" (Expression || ",")+ ")" ("as" Alias)?; + + //With open file statement + scope WithStatement implements Statement,ClassStatement = async:"async"? "with" (WithStatementContents || ",")+ ":" + StatementBlock ; WithStatementContents = Expression ("as" target:Name)? ; // target has slicing/etc - // variable declaration statement + //Variable related statements and definitions LocalVariableDeclarationStatement implements Statement = VariableDeclaration STATEMENT_END; VariableDeclaration implements Variable = Name ( @@ -203,126 +95,54 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit (":" TypeAnnotation) ); - GlobalVariableDeclaration implements Statement = "global" Name (":" TypeAnnotation)? STATEMENT_END; - - MultiVariableDeclaration implements Statement = (Name || ",")+ ","? "=" Expression STATEMENT_END; - ParenMultiVariableDeclaration implements Statement = "(" (Name || ",")+ ","? ")" "=" Expression STATEMENT_END; + GlobalVariableDeclaration implements Statement,ClassStatement = "global" names:(Name || ",")+ STATEMENT_END; + NonLocalVariableDeclaration implements Statement,ClassStatement = "nonlocal" names:(Name || ",")+ STATEMENT_END; + MultiVariableDeclaration implements Statement,ClassStatement = (Name || ",")+ ","? "=" (Expression || "," )+ ","? STATEMENT_END; + ParenMultiVariableDeclaration implements Statement,ClassStatement = "(" (Name || ",")+ ","? ")" "=" (Expression || "," )+ ","? STATEMENT_END; - interface VariableInit ; - SimpleInit implements VariableInit = Expression ; - // function declaration statement + //function declaration statement and function argument definitions interface FunctionDeclaration extends Function = Name ; - SimpleFunctionDeclaration implements FunctionDeclaration, Statement = PyDecorator* async:"async"? "def" Name "(" FunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" + SimpleFunctionDeclaration implements FunctionDeclaration, Statement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? "(" FunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock ; - FunctionParameters = (FunctionParameter || ",")* ","?; - - interface FunctionParameter; - SimpleFunctionParameter implements FunctionParameter, Variable = Name (":" TypeAnnotation)?; - OptionalFunctionParameter implements FunctionParameter, Variable = Name (":" TypeAnnotation)? "=" Expression ; - VarArgFunctionParameter implements FunctionParameter, Variable = "*" Name (":" TypeAnnotation)?; - KWArgFunctionParameter implements FunctionParameter, Variable = "**" Name (":" TypeAnnotation)?; - StarFunctionParameter implements FunctionParameter = "*"; - - @Override - Arguments = "(" - (Argument || ",")* - ","? - ")"; - - interface Argument; - NormalArgument implements Argument = Expression; - NamedArgument implements Argument = paramName:Name "=" Expression; - - PyDecorator = "@" Expression STATEMENT_END; + //Behaviour statements ReturnStatement implements Statement = "return" (Expression || ",")* ","? STATEMENT_END; YieldStatement implements Statement = "yield" (Expression || ",")* ","? STATEMENT_END; - RaiseStatement implements Statement = "raise" (Expression ("from" Name)?)? STATEMENT_END; - - ExpressionStatement implements Statement = Expression ("," Expression)* STATEMENT_END; + YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; //PEP380 + RaiseStatement implements Statement,ClassStatement = "raise" (Expression ("from" Name)?)? STATEMENT_END; - EmptyStatement implements Statement, ClassStatement = STATEMENT_END; - - MatchStatement implements Statement = key("match") Expression ":" MatchBlock; + //Typing related statements + TypeDeclarationStatement implements Statement = Expression ":" TypeAnnotation STATEMENT_END; + symbol TypeRuleStatement implements Statement,ClassStatement = key("type") Name GenericsAnnotation? "=" Expression STATEMENT_END; + TypeAliasStatement implements Statement,ClassStatement <400> = alias:Expression ":" "TypeAlias" "=" type:Expression STATEMENT_END; + ExpressionStatement implements Statement,ClassStatement = Expression ("," Expression)* STATEMENT_END; + //Match statement + MatchStatement implements Statement,ClassStatement = key("match") Expression ":" MatchBlock; scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; - CaseStatement = key("case") (Expression || "|")+ ("if" condition:Expression)? ":" StatementBlock; - - ConditionalExecutionStatement implements Statement = "if" condition: Expression ":" Expression; - - DeleteStatement implements Statement = "del" (Expression || ",")+ ","? STATEMENT_END; - - /*====================================== Expressions ======================================*/ - - SpreadListExpression implements Expression = "*" Expression; - SpreadMappingExpression implements Expression = "**" Expression; - splittoken "**"; - - // ternary-operator expression - TernaryOperatorExpression implements Expression <200> = thenExpression:Expression ( "if" condition:Expression - "else" elseExpression:Expression )+ ; - - //mathematical expression - IntegerDivisionExpression implements Expression <165>, InfixExpression = left:Expression operator:"//" right:Expression ; - IntegerPowExpression implements Expression <195>, InfixExpression = left:Expression operator:"**" right:Expression ; - MatrixMultiplicationExpression implements Expression <200>, InfixExpression = left:Expression operator:"@" right:Expression; - - //logical expressions - AndExpression implements Expression <120>, InfixExpression = left:Expression operator:"and" right:Expression ; - OrExpression implements Expression <117>, InfixExpression = left:Expression operator:"or" right:Expression ; - NotExpression implements Expression <10> = "not" Expression ; - IsExpression implements Expression <130>, InfixExpression = left:Expression operator:"is" right:Expression ; - InExpression implements Expression <195>, InfixExpression = left:Expression operator:"in" right:Expression ; - NotInExpression implements Expression <195>, InfixExpression = left:Expression operator:"not" "in" right:Expression; // TODO: set operator to "not in" programmatically - - //Bitwise expressions - BitwiseAndExpression implements Expression <120>, InfixExpression = left:Expression operator:"&" right:Expression; - BitwiseOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"|" right:Expression; - BitwiseXOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"^" right:Expression; - BitwiseLeftShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:"<<" right:Expression; - BitwiseRightShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:">>" right:Expression; - - BitwiseOnesComplimentExpression implements Expression <120> = "~" Expression; - - // lambda expression - scope LambdaExpression implements Expression = "lambda" FunctionParameters ":" Expression ; - AppliedLambdaExpression implements Expression = "(" LambdaExpression ")" "(" Expression ")" ; - - AwaitExpression implements Expression = "await" Expression; - - AnnotatedAssignmentExpression implements Expression <60> = - left:Expression - ":" annotated: TypeAnnotation - operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" - | "^=" | ">>=" | ">>>=" | "<<=" | "%=" ] - right:Expression; - - UnpackingAssignmentExpression implements Expression <60> = - "(" left:Expression ("," left:Expression)* ","? ")" "=" - right: Expression; - - // 6.3.3 - Slicing - IndexExpression implements Expression = Expression "[" (IndexExpressionInner || ",")+ tuple:","? "]"; - - // slice_item - interface IndexExpressionInner; - - SimpleIndex implements IndexExpressionInner = Expression; - ProperSlice implements IndexExpressionInner = lower:Expression? ":" upper:Expression? (":" stride:Expression?)?; - - // Walrus operator - PyAssignmentExpression implements Expression = variable:Name operator:":=" right:Expression; - - // List/Set/Dict comprehension - ListComprehensionExpression implements Expression = "[" Expression "for" ForControl GeneratorFilter? "]"; - SetComprehensionExpression implements Expression = "{" Expression "for" ForControl GeneratorFilter? "}"; - DictComprehensionExpression implements Expression = "{" Name ":" Expression "for" ForControl GeneratorFilter? "}"; - GeneratorExpression implements Expression = Expression "for" ForControl GeneratorFilter? ; - - GeneratorFilter = "if" condition:Expression; + scope CaseStatement = key("case") CasePattern ("if" condition:Expression)? ":" CaseStatementBlock; + CaseStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; + + // https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-patterns + CasePattern = OpenSequencePattern | Pattern; + Pattern = AsPattern | OrPattern; + AsPattern = OrPattern "as" Alias; + OrPattern = (ClosedPattern|| "|")+; + ClosedPattern = PyQualifiedName | "(" Pattern ")" | SequencePattern | Expression; + + SequencePattern = ("[" OpenSequencePattern? "]") | ("(" OpenSequencePattern? ")"); + OpenSequencePattern = (MaybeStarredPattern || ",")+ ","?; + MaybeStarredPattern = ("*"? Name) | Pattern; + + //Other Statements + DeleteStatement implements Statement,ClassStatement= "del" (Expression || ",")+ ","? STATEMENT_END; + LiteralStatement implements ClassStatement = Literal STATEMENT_END; + EmptyStatement implements Statement, ClassStatement = STATEMENT_END; + PyDecorator = "@" Expression STATEMENT_END; + symbol Alias = Name; /*===========================Classes======================================*/ @@ -330,30 +150,22 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit interface scope symbol PythonClass = Name ; // class declaration statement - ClassDeclaration implements PythonClass, Statement, ClassStatement = PyDecorator* "class" Name ( "(" ((superClasses:PyQualifiedName TypeAnnotation? | arguments:NamedArgument) ","?)* ")" )? ":" ClassStatementBlock ; + ClassDeclaration implements PythonClass, Statement, ClassStatement = PyDecorator* "class" Name GenericsAnnotation? + ( "(" ((superClasses:PyQualifiedName TypeAnnotation? | arguments:NamedArgument) ","?)* ")" )? ":" ClassStatementBlock ; interface ClassStatement; - ClassStatementBlock = BLOCK_START ClassStatementBlockBody BLOCK_END; + ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END)|ClassStatement; ClassStatementBlockBody = ClassStatement+; - ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; + ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? + "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; ClassFunctionParameters = (FunctionParameter || ",")* ","?; - ClassAttributes implements ClassStatement = VariableDeclaration STATEMENT_END; + ClassAttributes implements ClassStatement <300> = VariableDeclaration STATEMENT_END; ClassCommentStatement implements ClassStatement = MultiLineStringLiteral STATEMENT_END; - /*====================================== Type Annotations ======================================*/ - - interface TypeAnnotation; - StringTypeAnnotation implements TypeAnnotation = StringLiteralPython; - TupleTypeAnnotation implements TypeAnnotation = "(" (TypeAnnotation || ",")+ ","? ")"; - QualifiedTypeAnnotation implements TypeAnnotation = type:PyQualifiedName ("[" typeParams:(TypeAnnotation || ",")* "]")?; - AlternativeTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "|" rhs:TypeAnnotation; - CommaTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "," rhs:TypeAnnotation ","?; - ListTypeAnnotation implements TypeAnnotation = "[" TypeAnnotation "]"; - EllipsisTypeAnnotation implements TypeAnnotation = "..."; - ComplexTypeAnnotation implements TypeAnnotation <200> = Expression; + } diff --git a/src/main/grammars/de/monticore/PythonBasis.mc4 b/src/main/grammars/de/monticore/PythonBasis.mc4 new file mode 100644 index 0000000..3720b20 --- /dev/null +++ b/src/main/grammars/de/monticore/PythonBasis.mc4 @@ -0,0 +1,265 @@ +/* (c) https://github.com/MontiCore/monticore */ +package de.monticore; + +import de.monticore.MultilineString; +import de.monticore.expressions.*; +import de.monticore.literals.MCCommonLiterals; +import de.monticore.symbols.OOSymbols; + +grammar PythonBasis extends MultilineString, + MCCommonLiterals, + CommonExpressions, + AssignmentExpressions, + OOSymbols{ + + /*====================================== Tokens ======================================*/ + @Override + token WS = (' ' | '\t' | '\r' | '\n' ) : -> channel(HIDDEN); + @Override + token SL_COMMENT = "#" (~('\n' | '\r' ))* : -> channel(HIDDEN); + token ByteOrderMark = '\uFEFF' : -> skip; + + /** + * The following utf-8 symbols are used to parse the Python files without having to add whitespace to the Grammar. + * In a preprocessing step code blocks are denoted with \u204f = ⦃ and \u2984 = ⦄, and lines are ended with \u204f = ⁏ + */ + token BLOCK_START = '\u2983'; + token BLOCK_END = '\u2984'; + token STATEMENT_END = ';'? '\u204f' | ';' '\u204f'?; + + // Will be filtered out by the WhitespacePreprocessingTokenSource + // Used break a line without finishing the statement + token CONTINUE_LINE_TOKEN = '\\' '\r'? '\n'; + + // === string tokens for python === + //Often (mis)used as a multiline comment but can also be used as a string literal, thus we can not skip it + token MultiLineStringToken = ((("\'\'\'") .*? ("\'\'\'")) | ((DoubleQuoteMultilineStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); + token MultiLineFStringToken = ((("\'\'\'") .*? ("\'\'\'")) |((DoubleQuoteMultilineFStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); + + //Double quoted Strings "Text" + @Override + token String = '"' (StringDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; + + fragment token StringDQCharactersPython + = (StringDQCharacterPython)+; + fragment token StringDQCharacterPython + = ~ ('"'| '\\' | '\n') | PythonEscapeSequence; + + //Single quoted Strings 'Text' + token StringPython = '\'' (StringSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; + + fragment token StringSQCharactersPython + = (StringSQCharacterPython)+; + fragment token StringSQCharacterPython + = ~ ('\''| '\\' | '\n') | PythonEscapeSequence; + + //Escape in strings "\n" + fragment token PythonEscapeSequence = '\\' .; + + //Double and single quoted strings with an f modifier f'text1{exp}text2', separate definition to allow more expressions. + token FSQStringPython + = ('f'|'F') '\'' (StringFSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; + token FDQStringPython + = ('f'|'F') '"' (StringFDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; + + fragment token StringFSQCharactersPython + = (StringFSQCharacterPython)+; + fragment token StringFDQCharactersPython + = (StringFDQCharacterPython)+; + + fragment token StringFSQCharacterPython + = ~ ('\''| '\\')| PythonEscapeSequence; + fragment token StringFDQCharacterPython + = ~ ('"'| '\\') | PythonEscapeSequence; + + // === number tokens for python === + + token FloatWithExponent = (DigitsPart | PyFloat) ('e'|'E') ('+' | '-')? DigitsPart; + FloatWithExponentLiteral implements NumericLiteral <100> = FloatWithExponent; + + token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.'; + PyFloatLiteral implements NumericLiteral <200> = PyFloat; + + // PEP 515 + token DigitsPart = Digit ('_'? Digit)*; + + token HexNumberToken = '0' 'x' ('0'..'9' | 'a'..'f' | 'A'..'F')+; + // PEP 515: Underscores in Numeric Literals + @Override + token Digits + = Digit ('_'? Digit)*; // technically the first digit must be nonzero except for 0(_0)* + + /*====================================== Literals ======================================*/ + + HexNumberLiteral implements NumericLiteral <100> = HexNumberToken; + ImaginaryNumberLiteral implements NumericLiteral <100> = {noSpace(2)}? (DigitsPart| Digits | PyFloat) key("j"); + + //Literals for Datastructures + ArrayLiteral implements Literal = "[" (VariableInit || ",")* ","? "]" ; + TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ; + DictLiteral implements Literal = "{" (DictEntry || ",")* ","? "}" ; + SetLiteral implements Literal = "{" (Expression || ",")* ","? "}" ; + + DictEntry = key:VariableInit ":" value:VariableInit | SpreadMappingExpression; + + //Literals and helper-definitions regarding Strings + MultiLineStringLiteral implements Literal = (StringModifier)? MultiLineStringToken; + MultiLineFStringLiteral implements Literal = (StringModifier)? MultiLineFStringToken; + + //Char is necessary because single quoted characters will be recognized as char tokens not string tokens, + // as char is imported from MCCommonLiterals.mc4 + StringLiteralPython implements Literal, SignedLiteral = + ( + StringModifier? + (sourceStrPy:StringPython | sourceStr:String |sourceChar:Char) + ) + | fsource: FStringPython; + + StringsLiteralPython implements Literal <200> = (StringLiteralPython | StringLiteral | MultiLineStringLiteral | MultiLineFStringLiteral | FStringPython)+; + + FStringPython = (FSQStringPython | FDQStringPython | ("f"|"F") Char); + StringModifier = /*{cmpTokenRegEx(1, "(r|b|u|R|B|U)+")}?*/ type:Name; + + // boolean literals for python + @Override + BooleanLiteral implements Literal, SignedLiteral = + source:["True" | "False"]; + + // https://docs.python.org/dev/library/constants.html#Ellipsis + EllipsisLiteral implements Literal = "..."; + splittoken "..."; + + /*====================================== Expressions ======================================*/ + //Spreadlist expression + SpreadListExpression implements Expression = "*" Expression; + SpreadMappingExpression implements Expression = "**" Expression; + splittoken "**"; + + // ternary-operator expression + TernaryOperatorExpression implements Expression <200> = thenExpression:Expression ( "if" condition:Expression + "else" elseExpression:Expression )+ ; + + //mathematical expression + IntegerDivisionExpression implements Expression <165>, InfixExpression = left:Expression operator:"//" right:Expression ; + IntegerPowExpression implements Expression <195>, InfixExpression = left:Expression operator:"**" right:Expression ; + MatrixMultiplicationExpression implements Expression <200>, InfixExpression = left:Expression operator:"@" right:Expression; + + //logical expressions + AndExpression implements Expression <120>, InfixExpression = left:Expression operator:"and" right:Expression ; + OrExpression implements Expression <117>, InfixExpression = left:Expression operator:"or" right:Expression ; + NotExpression implements Expression <10> = "not" Expression ; + IsExpression implements Expression <130>, InfixExpression = left:Expression operator:"is" right:Expression ; + InExpression implements Expression <195>, InfixExpression = left:Expression operator:"in" right:Expression ; + NotInExpression implements Expression <195>, InfixExpression = left:Expression operator:"not" "in" right:Expression; // TODO: set operator to "not in" programmatically + + //Bitwise expressions + BitwiseAndExpression implements Expression <120>, InfixExpression = left:Expression operator:"&" right:Expression; + BitwiseOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"|" right:Expression; + BitwiseXOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"^" right:Expression; + BitwiseLeftShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:"<<" right:Expression; + BitwiseRightShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:">>" right:Expression; + + BitwiseOnesComplimentExpression implements Expression <120> = "~" Expression; + + // lambda expression + scope LambdaExpression implements Expression = "lambda" FunctionParameters ":" Expression ; + AppliedLambdaExpression implements Expression = "(" LambdaExpression ")" "(" Expression ")" ; + + //Assignment expressions + AnnotatedAssignmentExpression implements Expression <60> = + left:Expression + ":" annotated: TypeAnnotation + "=" right:Expression ","?; + @Override + AssignmentExpression implements Expression <60> = + left:Expression + operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" + | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="] + right:Expression ","?; + + UnpackingAssignmentExpression implements Expression <60> = + "(" left:Expression ("," left:Expression)* ","? ")" "=" + right: Expression; + + // 6.3.3 - Slicing + IndexExpression implements Expression = Expression "[" (IndexExpressionInner || ",")+ tuple:","? "]"; + + // slice_item + interface IndexExpressionInner; + + SimpleIndex implements IndexExpressionInner = Expression; + ProperSlice implements IndexExpressionInner = lower:Expression? ":" upper:Expression? (":" stride:Expression?)?; + + // Walrus operator + PyAssignmentExpression implements Expression = variable:Name operator:":=" right:Expression; + + //Await expression + AwaitExpression implements Expression = "await" Expression; + + // List/Set/Dict comprehension + ListComprehensionExpression implements Expression = "[" Expression "for" ForControl GeneratorFilter* "]"; + SetComprehensionExpression implements Expression = "{" Expression "for" ForControl GeneratorFilter* "}"; + DictComprehensionExpression implements Expression = "{" Name ":" Expression "for" ForControl GeneratorFilter* "}"; + GeneratorExpression implements Expression = Expression "for" ForControl GeneratorFilter? ; + + + ForControl = ForList "in" ForIterable; + + interface ForDecomposition; + ForList = (ForDecomposition || ",")+ ","?; + ForVariable implements Variable, ForDecomposition = Name; + ForDecompositionParenthesis implements ForDecomposition = "(" ForList? ")"; + ForDecompositionBrackets implements ForDecomposition = "[" ForList? "]"; + ForStarredVariable implements ForDecomposition = "*" ForDecomposition ; + ForPyQualifiedName implements ForDecomposition = PyQualifiedName; + ForIterable = Expression; + + GeneratorFilter = "if" condition:Expression; + + /*====================================== Type Annotations ======================================*/ + + interface TypeAnnotation; + GenericTypeAnnotation implements TypeAnnotation <100> = TypeAnnotation GenericsAnnotation; + StringTypeAnnotation implements TypeAnnotation <100> = StringLiteralPython; + TupleTypeAnnotation implements TypeAnnotation <100> = "(" (TypeAnnotation || ",")+ ","? ")"; + QualifiedTypeAnnotation implements TypeAnnotation <200> = type:PyQualifiedName; + AlternativeTypeAnnotation implements TypeAnnotation <100> = lhs:TypeAnnotation "|" rhs:TypeAnnotation; + CommaTypeAnnotation implements TypeAnnotation <60> = lhs:TypeAnnotation "," rhs:TypeAnnotation ","?; + ListTypeAnnotation implements TypeAnnotation <100> = "[" TypeAnnotation? "]"; + EllipsisTypeAnnotation implements TypeAnnotation <100> = "..."; + ParameterizedGenericTypeAnnotation implements TypeAnnotation <100> = TypeAnnotation "[" (Expression ||",")* ","? "]"; + + /*====================================== Generics ======================================*/ + + GenericsAnnotation = "[" Generics? ","? "]"; + Generics = (Generic || ",")+ ; + Generic = type:TypeAnnotation (":" TypeAnnotation)?; + + /*====================================== Variables ======================================*/ + + interface VariableInit ; + SimpleInit implements VariableInit = Expression ; + + //Helper-definition for qualified names in python. + PyQualifiedName = (Name || ".")+; + astrule PyQualifiedName = method public String joined(){ + return String.join(".", getNameList()); + }; + + FunctionParameters = (FunctionParameter || ",")* ","?; + interface FunctionParameter; + SimpleFunctionParameter implements FunctionParameter, Variable <100> = Name (":" TypeAnnotation)?; + OptionalFunctionParameter implements FunctionParameter, Variable <100> = Name (":" TypeAnnotation)? "=" Expression ; + VarArgFunctionParameter implements FunctionParameter, Variable <100> = "*" Name (":" TypeAnnotation)?; + KWArgFunctionParameter implements FunctionParameter, Variable <100> = "**" Name (":" TypeAnnotation)?; + StarFunctionParameter implements FunctionParameter <100> = "*"; + SlashFunctionParameter implements FunctionParameter <100> = "/"; + @Override + Arguments = "(" + (Argument || ",")* + ","? + ")"; + interface Argument; + NormalArgument implements Argument = Expression; + NamedArgument implements Argument = paramName:Name "=" Expression; + } diff --git a/src/main/java/de/monticore/python/_cocos/PythonFunctionArgumentSizeCoco.java b/src/main/java/de/monticore/python/_cocos/PythonFunctionArgumentSizeCoco.java index abf6af1..8893ea5 100644 --- a/src/main/java/de/monticore/python/_cocos/PythonFunctionArgumentSizeCoco.java +++ b/src/main/java/de/monticore/python/_cocos/PythonFunctionArgumentSizeCoco.java @@ -5,9 +5,9 @@ import de.monticore.expressions.commonexpressions._cocos.CommonExpressionsASTCallExpressionCoCo; import de.monticore.expressions.expressionsbasis._ast.ASTExpression; import de.monticore.expressions.expressionsbasis._ast.ASTNameExpression; -import de.monticore.python._ast.ASTArgument; -import de.monticore.python._ast.ASTArguments; -import de.monticore.python._ast.ASTOptionalFunctionParameter; +import de.monticore.pythonbasis._ast.ASTArgument; +import de.monticore.pythonbasis._ast.ASTArguments; +import de.monticore.pythonbasis._ast.ASTOptionalFunctionParameter; import de.monticore.python._symboltable.IPythonScope; import de.monticore.symbols.basicsymbols._symboltable.FunctionSymbol; import de.se_rwth.commons.logging.Log; diff --git a/src/main/java/de/monticore/python/_cocos/PythonFunctionDuplicateParameterNameCoco.java b/src/main/java/de/monticore/python/_cocos/PythonFunctionDuplicateParameterNameCoco.java index 3931e95..3551691 100644 --- a/src/main/java/de/monticore/python/_cocos/PythonFunctionDuplicateParameterNameCoco.java +++ b/src/main/java/de/monticore/python/_cocos/PythonFunctionDuplicateParameterNameCoco.java @@ -2,9 +2,9 @@ import de.monticore.python._ast.ASTClassFunctionDeclaration; import de.monticore.python._ast.ASTFunctionDeclaration; -import de.monticore.python._ast.ASTFunctionParameter; +import de.monticore.pythonbasis._ast.ASTFunctionParameter; import de.monticore.python._ast.ASTSimpleFunctionDeclaration; -import de.monticore.python._util.PythonTypeDispatcher; +import de.monticore.pythonbasis._util.PythonBasisTypeDispatcher; import de.se_rwth.commons.logging.Log; import java.util.ArrayList; @@ -26,14 +26,14 @@ public void check(ASTFunctionDeclaration node) { parameters.addAll(((ASTClassFunctionDeclaration) node).getClassFunctionParameters().getFunctionParameterList()); } - PythonTypeDispatcher td = new PythonTypeDispatcher(); + PythonBasisTypeDispatcher td = new PythonBasisTypeDispatcher(); for (ASTFunctionParameter parameter : parameters) { String name = null; - if(td.isBasicSymbolsASTTypeVar(parameter)) { + if(td.isBasicSymbolsASTTypeVar(parameter)) { name = td.asBasicSymbolsASTTypeVar(parameter).getName(); - }else if(td.isPythonASTSimpleFunctionParameter(parameter)){ - name = td.asPythonASTSimpleFunctionParameter(parameter).getName(); + }else if(td.isPythonBasisASTSimpleFunctionParameter(parameter)) { + name = td.asPythonBasisASTSimpleFunctionParameter(parameter).getName(); } if (name != null) { if (names.contains(name)) { diff --git a/src/main/java/de/monticore/python/_cocos/PythonLambdaDuplicateParameterNameCoco.java b/src/main/java/de/monticore/python/_cocos/PythonLambdaDuplicateParameterNameCoco.java deleted file mode 100644 index 724ab03..0000000 --- a/src/main/java/de/monticore/python/_cocos/PythonLambdaDuplicateParameterNameCoco.java +++ /dev/null @@ -1,31 +0,0 @@ -package de.monticore.python._cocos; - -import de.monticore.python._ast.ASTFunctionParameter; -import de.monticore.python._ast.ASTLambdaExpression; -import de.monticore.python._util.PythonTypeDispatcher; -import de.se_rwth.commons.logging.Log; - -import java.util.HashSet; -import java.util.Set; - -public class PythonLambdaDuplicateParameterNameCoco implements PythonASTLambdaExpressionCoCo { - - @Override - public void check(ASTLambdaExpression node) { - Set names = new HashSet<>(); - - PythonTypeDispatcher td = new PythonTypeDispatcher(); - - for (ASTFunctionParameter parameter : node.getFunctionParameters().getFunctionParameterList()) { - if(td.isBasicSymbolsASTVariable(parameter)) { - String name = td.asBasicSymbolsASTVariable(parameter).getName(); - if (names.contains(name)) { - Log.error("Duplicate parameter name '" + name + "' in lambda function " + node.get_SourcePositionStart()); - } else { - names.add(name); - } - } - } - - } -} diff --git a/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java b/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java index 18b9afd..9169e66 100644 --- a/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java +++ b/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java @@ -23,6 +23,7 @@ public class StateBasedWhitespacePreprocessingTokenSource { private final List closingParens; private Token lastToken; private Token lastEmittedToken; + private PreprocessingTokens preprocessingTokens; public StateBasedWhitespacePreprocessingTokenSource( Pair source, @@ -50,6 +51,7 @@ public StateBasedWhitespacePreprocessingTokenSource( this.lastToken = null; this.lastEmittedToken = null; + this.preprocessingTokens = preprocessingTokens; } public List process(Token token) { @@ -70,8 +72,12 @@ public List process(Token token) { // whitespace sensitive res = sensitiveProcessor.process(token, lastToken, lastEmittedToken); } else { - // whitespace insensitive - return List.of(token); + // always ignore the continueLineToken + if(token.getType() == preprocessingTokens.continueLineTokenType){ + return List.of(); + } else { + return List.of(token); + } } // bookkeeping of emitted tokens diff --git a/src/main/java/de/monticore/pythonbasis/_cocos/PythonBasisLambdaDuplicateParameterNameCoco.java b/src/main/java/de/monticore/pythonbasis/_cocos/PythonBasisLambdaDuplicateParameterNameCoco.java new file mode 100644 index 0000000..46f3606 --- /dev/null +++ b/src/main/java/de/monticore/pythonbasis/_cocos/PythonBasisLambdaDuplicateParameterNameCoco.java @@ -0,0 +1,36 @@ +/* (c) https://github.com/MontiCore/monticore */ +package de.monticore.pythonbasis._cocos; + +import de.monticore.pythonbasis._ast.ASTFunctionParameter; +import de.monticore.pythonbasis._ast.ASTLambdaExpression; +import de.monticore.pythonbasis._util.PythonBasisTypeDispatcher; +import de.se_rwth.commons.logging.Log; + +import java.util.HashSet; +import java.util.Set; + +public class PythonBasisLambdaDuplicateParameterNameCoco implements + PythonBasisASTLambdaExpressionCoCo { + + @Override + public void check(ASTLambdaExpression node) { + Set names = new HashSet<>(); + + PythonBasisTypeDispatcher td = new PythonBasisTypeDispatcher(); + + for (ASTFunctionParameter parameter : node.getFunctionParameters().getFunctionParameterList()) { + if (td.isBasicSymbolsASTVariable(parameter)) { + String name = td.asBasicSymbolsASTVariable(parameter).getName(); + if (names.contains(name)) { + Log.error("Duplicate parameter name '" + name + "' in lambda function " + node + .get_SourcePositionStart()); + } + else { + names.add(name); + } + } + } + + } + +} diff --git a/src/main/java/de/monticore/pythonbasis/_prettyprint/PythonBasisPrettyPrinter.java b/src/main/java/de/monticore/pythonbasis/_prettyprint/PythonBasisPrettyPrinter.java new file mode 100644 index 0000000..80bd19b --- /dev/null +++ b/src/main/java/de/monticore/pythonbasis/_prettyprint/PythonBasisPrettyPrinter.java @@ -0,0 +1,87 @@ +/* (c) https://github.com/MontiCore/monticore */ +package de.monticore.pythonbasis._prettyprint; + +// Generated code that was fixed manually, reoved generated comments. +public class PythonBasisPrettyPrinter extends PythonBasisPrettyPrinterTOP { + + public PythonBasisPrettyPrinter(de.monticore.prettyprint.IndentPrinter printer, + boolean printComments) { + super(printer, printComments); + } + + @Override + public void handle(de.monticore.pythonbasis._ast.ASTFStringPython node) { + if (this.isPrintComments()) { + de.monticore.prettyprint.CommentPrettyPrinter.printPreComments(node, getPrinter()); + } + if (node.isPresentFSQStringPython()) { + getPrinter().print("f" + node.getFSQStringPython() + "\'"); + } else if (node.isPresentFDQStringPython()) { + getPrinter().print("f" + node.getFDQStringPython() + "\" "); + } else if (node.isPresentChar()) { + getPrinter().print("f \'" + node.getChar() + "\'" + " "); + + } + if (this.isPrintComments()) { + de.monticore.prettyprint.CommentPrettyPrinter.printPostComments(node, getPrinter()); + } + } + + @Override + public void handle(de.monticore.pythonbasis._ast.ASTStringLiteralPython node) { + if (this.isPrintComments()) { + de.monticore.prettyprint.CommentPrettyPrinter.printPreComments(node, getPrinter()); + } + if (((node.isPresentSourceStrPy() || node.isPresentSourceStr()) || node.isPresentSourceChar())) { + + if (node.isPresentStringModifier()) { + node.getStringModifier().accept(getTraverser()); + } + if (node.isPresentSourceStrPy()) { + getPrinter().print("\'" + node.getSourceStrPy() + "\'"); + + } else if (node.isPresentSourceStr()) { + getPrinter().print( "\"" + node.getSourceStr() + "\"" + " "); + + } else if (node.isPresentSourceChar()) { + getPrinter().print( "\'" + node.getSourceChar() + "\'" + " "); + } + } + else if ((node.isPresentFsource() && !(((node.isPresentStringModifier() || node + .isPresentSourceStrPy()) || node.isPresentSourceStr()) || node.isPresentSourceChar()))) { + node.getFsource().accept(getTraverser()); + + } + if (this.isPrintComments()) { + de.monticore.prettyprint.CommentPrettyPrinter.printPostComments(node, getPrinter()); + } + + } + + @Override + public void handle(de.monticore.pythonbasis._ast.ASTArguments node) { + if (this.isPrintComments()) { + de.monticore.prettyprint.CommentPrettyPrinter.printPreComments(node, getPrinter()); + } + java.util.Iterator iter_argument = node + .getArgumentList().iterator(); + getPrinter().stripTrailing(); + getPrinter().print("("); + + if (iter_argument.hasNext()) { + iter_argument.next().accept(getTraverser()); + while (iter_argument.hasNext()) { + getPrinter().stripTrailing(); + getPrinter().print(","); + iter_argument.next().accept(getTraverser()); + } + } + getPrinter().stripTrailing(); + getPrinter().print(")"); + if (this.isPrintComments()) { + de.monticore.prettyprint.CommentPrettyPrinter.printPostComments(node, getPrinter()); + } + + } + +} diff --git a/src/test/java/de/monticore/python/ParseTest.java b/src/test/java/de/monticore/python/ParseTest.java index d6940c5..eeaeeb7 100644 --- a/src/test/java/de/monticore/python/ParseTest.java +++ b/src/test/java/de/monticore/python/ParseTest.java @@ -2,7 +2,7 @@ import de.monticore.expressions.expressionsbasis._ast.ASTLiteralExpression; import de.monticore.python._ast.ASTPythonScript; -import de.monticore.python._ast.ASTStringLiteralPython; +import de.monticore.pythonbasis._ast.ASTStringLiteralPython; import de.monticore.python._parser.PythonParser; import de.se_rwth.commons.logging.Log; import org.antlr.v4.runtime.Token; diff --git a/src/test/java/de/monticore/python/PythonTest.java b/src/test/java/de/monticore/python/PythonTest.java index 904b01d..07883bb 100644 --- a/src/test/java/de/monticore/python/PythonTest.java +++ b/src/test/java/de/monticore/python/PythonTest.java @@ -40,6 +40,8 @@ public void parseFloorDiv(){ @Test public void parseListComprehension(){ parseModelFromStringAndExpectSuccess("rgb = [i for i in range(0, hlen, hlen // 3)]\n"); + parseModelFromStringAndExpectSuccess("rgb = [i for i, *b in range(0, hlen, hlen // 3) if cond1 if cond2]\n"); + } @Test @@ -231,7 +233,31 @@ public void parseInvalidForLoopStatement() { " print(x)\n" ); } - + @Test + public void parseValidMatchStatement(){ + parseModelFromStringAndExpectSuccess( + "match x:\n" + + " case (a,b as var1) as var2: pass\n" + + " case (a,b,c) as var3: pass\n" + ); + parseModelFromStringAndExpectSuccess( + "match x:\n" + + " case a as var1: pass\n" + + " case (b,c) as var2: pass\n" + ); + } + @Test + public void parseInvalidMatchStatement(){ + parseModelFromStringAndExpectFail( + "match x:\n" + + " case (a,b as var1 ) as var2: pass\n" + + " case as var3: pass\n" + ); + parseModelFromStringAndExpectFail( + "match x:\n" + + " case: pass\n" + ); + } //valid while statements @Test public void parseValidWhileLoopStatement() { @@ -326,7 +352,13 @@ public void parseValidTryExceptStatements() { "finally:\n" + " print(\"Done\")\n" ); - + //Additional test inspired by https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py + parseModelFromStringAndExpectSuccess( + "try:\n" + + " i = 1//0\n" + + "except ZeroDivisionError or CustomZeroDivisionError as exp:\n" + + " print(\"Can not divide by zero\")\n" + ); } //invalid try-except-finally statements @@ -349,7 +381,15 @@ public void parseInvalidTryExceptStatements() { "else:\n" + " print(\"Success\")\n" ); - + // missing aliased expression + parseModelFromStringAndExpectFail( + "try:\n" + + " i = 1//0\n" + + "except as exp:\n" + + " print(\"Can not divide by zero\")\n" + + "else:\n" + + " print(\"Success\")\n" + ); // duplicate finally parseModelFromStringAndExpectFail( "try:\n" + @@ -489,6 +529,12 @@ public void parseValidLambdaStatement() { parseModelFromStringAndExpectSuccess("lambda: 1\n"); parseModelFromStringAndExpectSuccess("lambda x: x\n"); parseModelFromStringAndExpectSuccess("lambda x, y: x + y\n"); + //Additional tests inspired by https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py + parseModelFromStringAndExpectSuccess("lambda *a : 1\n"); + parseModelFromStringAndExpectSuccess("lambda **a : 1\n"); + parseModelFromStringAndExpectSuccess("lambda *a, **b : 1\n"); + parseModelFromStringAndExpectSuccess("lambda a=name : 1\n"); + parseModelFromStringAndExpectSuccess("lambda a=name, b= q+1 : 1\n"); } //invalid lambda statement @@ -499,20 +545,33 @@ public void parseInvalidLambdaStatement() { parseModelFromStringAndExpectFail("lambda x, y z\n"); } - // valid lambda statement + // valid raise statement @Test public void parseValidRaiseStatement() { parseModelFromStringAndExpectSuccess("raise RuntimeError('Error')\n"); parseModelFromStringAndExpectSuccess("raise\n"); } - //invalid lambda statement + //invalid raise statement @Test public void parseInvalidRaiseStatement() { parseModelFromStringAndExpectFail("raise RuntimeError('Error'), ArithmeticError('Error')\n"); parseModelFromStringAndExpectFail("raise RuntimeError('Error') ArithmeticError('Error')\n"); } + // tests for yield inspired by https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py + // valid yield statement + @Test + public void parseValidYieldStatement() { + parseModelFromStringAndExpectSuccess("def function():\n i+=1 \n yield i\n"); + parseModelFromStringAndExpectSuccess("def function():\n yield from anotherFunction()\n"); + } + //invalid yield statement + @Test + public void parseInvalidYieldStatement() { + parseModelFromStringAndExpectFail("def function():\n yield yield\n"); + parseModelFromStringAndExpectFail("def function():\n yield from\n"); + } /*===========================Literals======================================*/ // valid string literals python @@ -530,6 +589,21 @@ public void parseInvalidStringPython() { parseModelFromStringAndExpectFail("helloworld = Hello World\n"); } + //Same tests as above just with the modifier f (and with a single char string) + // valid fstring literals python + @Test + public void parseValidFStringPython() { + parseModelFromStringAndExpectSuccess("helloworld = f\" \"\n"); + parseModelFromStringAndExpectSuccess("helloworld = F'Hello World'\n"); + } + + // invalid fstring literals python + @Test + public void parseInvalidFStringPython() { + parseModelFromStringAndExpectFail("helloworld = f\"Hello World\n"); + parseModelFromStringAndExpectFail("helloworld = F'Hello World\n"); + parseModelFromStringAndExpectFail("helloworld = Hello World\n"); + } // boolean literals for python @Test public void parseValidBooleanPython() { @@ -564,6 +638,16 @@ public void parseInvalidTernaryOperator() { parseModelFromStringAndExpectFail("x = u if a==b else\n"); } + // tests for Walrus inspired by https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py + @Test + public void parseValidWalrusOperator(){ + parseModelFromStringAndExpectSuccess("def function():\n yield a:=2\n"); + } + @Test + public void parseInalidWalrusOperator(){ + parseModelFromStringAndExpectFail("def function():\n a:= Yield 2\n"); + } + // valid logical expressions @Test public void parseValidLogicalExpressions() { @@ -638,15 +722,6 @@ public void parseInvalidClassDeclaration() { " e.count+=1\n" + " self.list_x.append(i)\n" ); - //for loop in class - parseModelFromStringAndExpectFail( - "class myClass:\n" + - " def function_name(x,y):\n" + - " print(x,y)\n" + - " for i in range(4):\n" + - " print(i)\n" - ); - } /*===========================Other======================================*/