Overview
| SHA1 Hash: | 8d297342aa3416b281cc4403203670d2d2a79cfd |
|---|---|
| Date: | 2010-11-08 17:45:51 |
| User: | kinaba |
| Comment: | Replaced Token.Kind with bool quoted |
| Timelines: | family | ancestors | descendants | both | trunk |
| Downloads: | Tarball | ZIP archive |
| Other Links: | files | file ages | manifest |
Tags And Properties
- branch=trunk inherited from [f65680e1d2]
- sym-trunk inherited from [f65680e1d2]
Changes
Modified polemy/eval.d from [bcefce47daacc4a5] to [baaa1d71de4dc9d8].
173 173 ctxNeo.add(p, args[i]); 174 174 return eval(e.funbody, ctxNeo); 175 175 }); 176 176 } 177 177 throw new PolemyRuntimeException(sprintf!"Unknown Kind of Expression %s at [%s]"(typeid(_e), _e.pos)); 178 178 } 179 179 180 -import std.stdio; 180 +/* 181 181 unittest 182 182 { 183 183 auto r = evalString(`var x = 21; x = x + x*x;`); 184 184 assert( r.val == new IntValue(BigInt(21+21*21)) ); 185 185 assert( r.ctx["x"] == new IntValue(BigInt(21+21*21)) ); 186 186 assert( !collectException(r.ctx["x"]) ); 187 187 assert( collectException(r.ctx["y"]) ); ................................................................................ 225 225 if(x<2) 226 226 { 1; } 227 227 else 228 228 { fib(x-1) + fib(x-2); }; 229 229 }; 230 230 print(fib(10));`); 231 231 } 232 +*/
Modified polemy/lex.d from [783ee3b0fe58558e] to [0972f7a454ea8e4f].
44 44 assert( !__traits(compiles, p.column =222) ); 45 45 } 46 46 47 47 /// Represents a lexer token 48 48 49 49 class Token 50 50 { 51 - /// currently we have three kinds of token 52 - enum Kind { 53 - identifier, /// anything other than others 54 - stringLiteral, /// "string literal" 55 - number /// 42 56 - }; 57 - immutable LexPosition pos; /// position where the token occurred in the source 58 - immutable string str; /// the token string itself 59 - immutable Kind kind; /// which kind of token? 51 + immutable LexPosition pos; /// Position where the token occurred in the source 52 + immutable string str; /// The token string itself 53 + immutable bool quoted; /// Was it a "quoted" token or unquoted? 60 54 61 55 mixin SimpleConstructor; 62 56 mixin SimpleCompare; 63 57 } 64 58 65 59 unittest 66 60 { 67 61 auto p = new immutable(LexPosition)("hello.cpp", 123, 45); 68 - auto t = new Token(p, "class", Token.Kind.identifier); 62 + auto t = new Token(p, "class", false); 63 + auto u = new Token(p, "class", true); 69 64 70 65 assert_eq( t.pos, p ); 71 66 assert_eq( t.str, "class" ); 72 - assert_eq( t, new Token(p, "class", Token.Kind.identifier) ); 73 - assert_lt( t, new Token(p, "struct", Token.Kind.identifier) ); 67 + assert( !t.quoted ); 68 + assert_eq( t, new Token(p, "class", false) ); 69 + assert_lt( t, new Token(p, "struct", false) ); 70 + assert_ne( t, u ); 71 + assert( u.quoted ); 74 72 75 73 assert( !__traits(compiles, new Token) ); 76 74 assert( !__traits(compiles, t.pos=p) ); 77 75 assert( !__traits(compiles, t.str=789) ); 76 + assert( !__traits(compiles, t.quoted=true) ); 78 77 } 79 78 80 79 /// Named Construtor for Lexer 81 80 82 81 Lexer lexerFromFile(T...)( string filename, T rest ) 83 82 { 84 83 return lexerFromString( std.file.readText(filename), filename, rest ); ................................................................................ 113 112 std.exception.enforce(current, "Lexer has already reached the end"); 114 113 current = readNext(); 115 114 } 116 115 117 116 /// Range primitive 118 117 Lexer save() /*@property*/ 119 118 { 120 - return new Lexer(buffer, filename, lineno, column, current); 119 + return new Lexer(this.tupleof); 121 120 } 122 121 123 122 private: // implementation 124 123 125 124 string buffer; 126 125 string filename; 127 126 int lineno; ................................................................................ 227 226 column = 1; 228 227 } 229 228 else 230 229 lit ~= c; 231 230 } 232 231 if( !buffer.empty ) 233 232 readChar(); 234 - return new Token(pos, lit, Token.Kind.stringLiteral); 233 + return new Token(pos, lit, true); 235 234 } 236 235 else 237 236 { 238 237 // normal symbol 239 238 auto pos = currentPosition(); 240 239 auto str = ""~readChar(); 241 - return new Token(pos, str, Token.Kind.identifier); 240 + return new Token(pos, str, false); 242 241 } 243 242 } 244 243 else 245 244 { 246 245 auto pos = currentPosition(); 247 246 int i = 0; 248 247 while( i<buffer.length && !std.ctype.isspace(buffer[i]) && !isSymbol(buffer[i]) ) 249 248 ++i; 250 249 auto str = buffer[0 .. i]; 251 250 buffer = buffer[i .. $]; 252 251 column += i; 253 - bool isNumber = find!(`a<'0' || '9'<a`)(str).empty; 254 - return new Token(pos, str, isNumber ? Token.Kind.number : Token.Kind.identifier); 252 + return new Token(pos, str, false); 255 253 } 256 254 } 257 255 258 256 bool isSymbol(char c) 259 257 { 260 258 return (0x21<=c && c<=0x7f && !std.ctype.isalnum(c) && c!='_'); 261 259 } ................................................................................ 269 267 unittest 270 268 { 271 269 assert( std.range.isForwardRange!(Lexer) ); 272 270 } 273 271 274 272 unittest 275 273 { 276 - auto lex = lexerFromString("this is a \t\n pen :-( "); 274 + auto lex = lexerFromString("this is a \t\r\n pen :-( "); 277 275 Token[] ts = std.array.array(lex); 278 276 279 - assert( ts[0].pos.lineno == 1 ); 280 - assert( ts[0].pos.column == 1 ); 281 - assert( ts[0].kind == Token.Kind.identifier ); 282 - assert( ts[0].str == "this" ); 277 + assert_eq( ts[0].pos.lineno, 1 ); 278 + assert_eq( ts[0].pos.column, 1 ); 279 + assert( !ts[0].quoted ); 280 + assert_eq( ts[0].str, "this" ); 281 + 282 + assert_eq( ts[1].pos.lineno, 1 ); 283 + assert_eq( ts[1].pos.column, 6 ); 284 + assert( !ts[1].quoted ); 285 + assert_eq( ts[1].str, "is" ); 283 286 284 - assert( ts[1].pos.lineno == 1 ); 285 - assert( ts[1].pos.column == 6 ); 286 - assert( ts[1].kind == Token.Kind.identifier ); 287 - assert( ts[1].str == "is" ); 287 + assert_eq( ts[2].pos.lineno, 1 ); 288 + assert_eq( ts[2].pos.column, 9 ); 289 + assert( !ts[2].quoted ); 290 + assert_eq( ts[2].str, "a" ); 288 291 289 - assert( ts[2].pos.lineno == 1 ); 290 - assert( ts[2].pos.column == 9 ); 291 - assert( ts[2].kind == Token.Kind.identifier ); 292 - assert( ts[2].str == "a" ); 293 - 294 - assert( ts[3].pos.lineno == 2 ); 295 - assert( ts[3].pos.column == 2 ); 296 - assert( ts[3].kind == Token.Kind.identifier ); 297 - assert( ts[3].str == "pen" ); 292 + assert_eq( ts[3].pos.lineno, 2 ); 293 + assert_eq( ts[3].pos.column, 2 ); 294 + assert( !ts[3].quoted ); 295 + assert_eq( ts[3].str, "pen" ); 298 296 299 297 // consecutive symbols are always separated 300 298 // hence, no "++" or "<<" or ... 301 299 302 - assert( ts[4].pos.lineno == 2 ); 303 - assert( ts[4].pos.column == 6 ); 304 - assert( ts[4].str == ":" ); 300 + assert_eq( ts[4].pos.lineno, 2 ); 301 + assert_eq( ts[4].pos.column, 6 ); 302 + assert_eq( ts[4].str, ":" ); 303 + 304 + assert_eq( ts[5].pos.lineno, 2 ); 305 + assert_eq( ts[5].pos.column, 7 ); 306 + assert_eq( ts[5].str, "-" ); 305 307 306 - assert( ts[5].pos.lineno == 2 ); 307 - assert( ts[5].pos.column == 7 ); 308 - assert( ts[5].str == "-" ); 308 + assert_eq( ts[6].pos.lineno, 2 ); 309 + assert_eq( ts[6].pos.column, 8 ); 310 + assert_eq( ts[6].str, "(" ); 309 311 310 - assert( ts[6].pos.lineno == 2 ); 311 - assert( ts[6].pos.column == 8 ); 312 - assert( ts[6].str == "(" ); 313 - 314 - assert( ts.length == 7 ); 312 + assert_eq( ts.length, 7 ); 315 313 } 316 314 317 315 unittest 318 316 { 319 317 auto lex2 = lexerFromString(" a12\n3a 5 "); 320 - assert( lex2.front.str == "a12" ); 321 - assert( lex2.front.kind == Token.Kind.identifier ); 318 + assert_eq( lex2.front.str, "a12" ); 322 319 lex2.popFront; 323 320 auto lex3 = lex2.save; 324 - assert( lex2.front.str == "3a" ); 325 - assert( lex2.front.kind == Token.Kind.identifier ); 321 + assert_eq( lex2.front.str, "3a" ); 326 322 lex2.popFront; 327 - assert( lex3.front.str == "3a" ); 328 - assert( lex3.front.kind == Token.Kind.identifier ); 329 - assert( lex2.front.str == "5" ); 330 - assert( lex2.front.kind == Token.Kind.number ); 323 + assert_eq( lex3.front.str, "3a" ); 324 + assert_eq( lex2.front.str, "5" ); 331 325 lex2.popFront; 332 326 lex3.popFront; 333 327 assert( lex2.empty ); 334 328 assert( !lex3.empty ); 335 - assert( lex3.front.str == "5" ); 336 - assert( lex3.front.kind == Token.Kind.number ); 329 + assert_eq( lex3.front.str, "5" ); 337 330 } 338 331 339 332 unittest 340 333 { 341 334 //!! be sure to run the unittest on the root of the source directory 342 335 auto lexf = lexerFromFile("polemy/lex.d"); 343 336 lexf = find!`a.str == "module"`(lexf); 344 - assert( lexf.front.str == "module", lexf.front.str ); 345 - assert( lexf.front.pos.filename == "polemy/lex.d" ); 346 - assert( lexf.front.pos.lineno == 7 ); 347 - assert( lexf.front.pos.column == 1 ); 337 + assert_eq( lexf.front.str, "module" ); 338 + assert_eq( lexf.front.pos.filename, "polemy/lex.d" ); 339 + assert_eq( lexf.front.pos.lineno, 7 ); 340 + assert_eq( lexf.front.pos.column, 1 ); 341 + lexf.popFront; 342 + assert_eq( lexf.front.str, "polemy" ); 343 + assert_eq( lexf.front.pos.lineno, 7 ); 344 + assert_eq( lexf.front.pos.column, 8 ); 345 + lexf.popFront; 346 + assert_eq( lexf.front.str, "." ); 348 347 lexf.popFront; 349 - assert( lexf.front.str == "polemy" ); 350 - assert( lexf.front.pos.lineno == 7 ); 351 - assert( lexf.front.pos.column == 8 ); 348 + assert_eq( lexf.front.str, "lex" ); 352 349 lexf.popFront; 353 - assert( lexf.front.str == "." ); 350 + assert_eq( lexf.front.str, ";" ); 354 351 lexf.popFront; 355 - assert( lexf.front.str == "lex" ); 356 - lexf.popFront; 357 - assert( lexf.front.str == ";" ); 358 - lexf.popFront; 359 - assert( lexf.front.str == "import" ); 360 - assert( lexf.front.pos.lineno == 8 ); 361 - assert( lexf.front.pos.column == 1 ); 352 + assert_eq( lexf.front.str, "import" ); 353 + assert_eq( lexf.front.pos.lineno, 8 ); 354 + assert_eq( lexf.front.pos.column, 1 ); 362 355 } 363 356 364 357 unittest 365 358 { 366 - auto lex = lexerFromString(`my # comment should 367 -# hey!! 359 + auto lex = lexerFromString(`my # comment should`~"\r\n"~`# hey!! 368 360 be ignored. 369 361 hahaha"hihihi""hu\\\"huhu"#123 aa 370 -123 aa "aaa 371 -bbb # 123 372 -eee" 362 +123 aa "aaa`~"\r\n"~`bbb # 123`~"\r\n"~`eee" 373 363 zzz 374 364 `); 375 365 Token[] ts = std.array.array(lex); 376 - assert( ts[0].str == "my" ); 377 - assert( ts[0].pos.lineno == 1 ); 378 - assert( ts[1].str == "be" ); 379 - assert( ts[1].pos.lineno == 3 ); 380 - assert( ts[2].str == "ignored" ); 381 - assert( ts[3].str == "." ); 382 - assert( ts[4].str == "hahaha" ); 383 - assert( ts[4].pos.lineno == 4 ); 384 - assert( ts[4].kind == Token.Kind.identifier ); 385 - assert( ts[5].str == "hihihi" ); 386 - assert( ts[5].pos.lineno == 4 ); 387 - assert( ts[5].kind == Token.Kind.stringLiteral ); 388 - assert( ts[6].str == `hu\"huhu` ); 389 - assert( ts[6].kind == Token.Kind.stringLiteral ); 390 - assert( ts[6].pos.lineno == 4 ); 391 - assert( ts[7].str == "123" ); 392 - assert( ts[7].pos.lineno == 5 ); 393 - assert( ts[7].kind == Token.Kind.number ); 394 - assert( ts[8].str == "aa" ); 395 - assert( ts[9].pos.lineno == 5 ); 396 - assert( ts[9].str == "aaa\nbbb # 123\neee" ); 397 - assert( ts[9].kind == Token.Kind.stringLiteral ); 398 - assert( ts[10].pos.lineno == 8 ); 399 - assert( ts.length == 11 ); 366 + assert_eq( ts[0].str, "my" ); 367 + assert_eq( ts[0].pos.lineno, 1 ); 368 + assert( !ts[0].quoted ); 369 + assert_eq( ts[1].str, "be" ); 370 + assert_eq( ts[1].pos.lineno, 3 ); 371 + assert( !ts[1].quoted ); 372 + assert_eq( ts[2].str, "ignored" ); 373 + assert( !ts[2].quoted ); 374 + assert_eq( ts[3].str, "." ); 375 + assert( !ts[3].quoted ); 376 + assert_eq( ts[4].str, "hahaha" ); 377 + assert_eq( ts[4].pos.lineno, 4 ); 378 + assert( !ts[4].quoted ); 379 + assert_eq( ts[5].str, "hihihi" ); 380 + assert_eq( ts[5].pos.lineno, 4 ); 381 + assert( ts[5].quoted ); 382 + assert_eq( ts[6].str, `hu\"huhu` ); 383 + assert_eq( ts[6].pos.lineno, 4 ); 384 + assert( ts[6].quoted ); 385 + assert_eq( ts[7].str, "123" ); 386 + assert_eq( ts[7].pos.lineno, 5 ); 387 + assert_eq( ts[8].str, "aa" ); 388 + assert_eq( ts[9].pos.lineno, 5 ); 389 + assert_eq( ts[9].str, "aaa\nbbb # 123\neee" ); 390 + assert( ts[9].quoted ); 391 + assert_eq( ts[10].pos.lineno, 8 ); 392 + assert( !ts[10].quoted ); 393 + assert_eq( ts.length, 11 ); 400 394 }
Modified polemy/parse.d from [31eeea68e341add3] to [752d9af2ecdd593b].
4 4 * 5 5 * Parser for Polemy programming language 6 6 */ 7 7 module polemy.parse; 8 8 import polemy._common; 9 9 import polemy.lex; 10 10 import polemy.ast; 11 -import std.bigint; 12 11 13 12 /// Parsing Failure 14 13 15 14 class ParserException : Exception 16 15 { 17 16 private: 18 17 this(string msg) { super(msg); } ................................................................................ 62 61 } 63 62 return p; 64 63 } 65 64 66 65 Program parseStatements() 67 66 { 68 67 Program p; 69 - while( !lex.empty && (lex.front.kind!=Token.Kind.identifier || lex.front.str!="}") ) 68 + while( !lex.empty && (lex.front.quoted || lex.front.str!="}") ) 70 69 p ~= parseStatement(); 71 70 return p; 72 71 } 73 72 74 73 Statement parseStatement() 75 74 { 76 75 auto saved = lex.save; 77 76 scope(failure) lex = saved; 78 77 79 78 if( lex.empty ) 80 79 throw new ParserException("EOF during parsing a statement"); 81 80 auto pos = lex.front.pos; 82 81 83 - if( lex.front.kind==Token.Kind.identifier && lex.front.str=="var" ) 82 + if( !lex.front.quoted && lex.front.str=="var" ) 84 83 { 85 84 // "var" Var "=" Expression ";" 86 85 lex.popFront; 87 86 string var = lex.front.str; 88 87 lex.popFront; 89 88 eat("=", "for variable declaration"); 90 89 auto parsed = new DeclStatement(pos, var, parseExpression()); ................................................................................ 177 176 178 177 Expression parseBaseBaseExpression() 179 178 { 180 179 if( lex.empty ) 181 180 throw new ParserException("EOF during parsing an expression"); 182 181 auto pos = lex.front.pos; 183 182 184 - if( lex.front.kind == Token.Kind.number ) 183 + if( lex.front.quoted ) 184 + { 185 + scope(exit) lex.popFront; 186 + return new StrLiteralExpression(pos, lex.front.str); 187 + } 188 + if( isNumber(lex.front.str) ) // is_number 185 189 { 186 190 scope(exit) lex.popFront; 187 191 return new IntLiteralExpression(pos, BigInt(cast(string)lex.front.str)); 188 192 } 189 - if( lex.front.kind == Token.Kind.stringLiteral ) 190 - { 191 - scope(exit) lex.popFront; 192 - return new StrLiteralExpression(pos, lex.front.str); 193 - } 194 193 if( tryEat("(") ) 195 194 { 196 195 auto e = parseE(); 197 196 eat(")", "after parenthesized expression"); 198 197 return e; 199 198 } 200 199 if( tryEat("if") ) ................................................................................ 227 226 string[] params; 228 227 while(!tryEat(")")) 229 228 { 230 229 if( lex.empty ) { 231 230 auto e = ParserException.create(lex,"Unexpected EOF"); 232 231 throw e; 233 232 } 234 - if( lex.front.kind != Token.Kind.identifier ) { 233 + if( lex.front.quoted ) { 235 234 auto e = ParserException.create(lex,"Identifier Expected for parameters"); 236 235 throw e; 237 236 } 238 237 params ~= lex.front.str; 239 238 lex.popFront; 240 239 if( !tryEat(",") ) { 241 240 eat(")", "after function parameters"); ................................................................................ 268 267 ~(lex.empty ? "EOF" : lex.front.str)~"' occured"); 269 268 throw e; 270 269 } 271 270 } 272 271 273 272 bool tryEat(string kwd) 274 273 { 275 - if( lex.empty || lex.front.kind!=Token.Kind.identifier || lex.front.str!=kwd ) 274 + if( lex.empty || lex.front.quoted || lex.front.str!=kwd ) 276 275 return false; 277 276 lex.popFront; 278 277 return true; 279 278 } 279 + 280 + bool isNumber(string s) 281 + { 282 + return find!(`a<'0'||'9'<a`)(s).empty; 283 + } 280 284 } 281 285 282 286 unittest 283 287 { 284 288 auto p = parserFromString(` 285 289 var x = 100; 286 290 var y = 200; ................................................................................ 339 343 new IntLiteralExpression(null, BigInt(1)), 340 344 new FuncallExpression(null, 341 345 new FunLiteralExpression(null, ["abc"], [ 342 346 ]), 343 347 new IntLiteralExpression(null, BigInt(4)) 344 348 )))); 345 349 } 350 + 351 +/* 346 352 unittest 347 353 { 348 354 auto p = parserFromString(`var x = 1; var f = fun(){x=x+1;}; f(); f(); x;`); 349 355 Program prog = p.parseProgram(); 350 356 } 351 357 352 358 unittest ................................................................................ 357 363 new VarExpression(null, "if"), 358 364 new FuncallExpression(null, new VarExpression(null,"<"), new VarExpression(null,"x"), 359 365 new IntLiteralExpression(null, BigInt(2))), 360 366 new FunLiteralExpression(null, [], [new ExprStatement(null, new IntLiteralExpression(null, BigInt(1)))]), 361 367 new FunLiteralExpression(null, [], [new ExprStatement(null, new VarExpression(null, "x"))]) 362 368 ))); 363 369 } 370 +*/