1 +/** 2 + * Authors: k.inaba 3 + * License: NYSL 0.9982 http://www.kmonos.net/nysl/ 4 + * 5 + * Lexer for Polemy programming language. 6 + */ 1 7 module polemy.lex; 2 8 import polemy._common; 3 -/* 4 - * Author: k.inaba 5 - * License: NYSL 0.9982 (http://www.kmonos.net/nysl/ 6 - * Lexer for the polemy programming language 7 - */ 8 9 9 10 import std.file : readText; 10 11 import std.string : munch; 11 12 import std.ctype; 12 13 13 14 /// Represents a position in a source code 14 15 15 16 class LexPosition 16 17 { 17 - immutable string filename; ///< name of the source file 18 - immutable int lineno; ///< line number: 1, 2, ... 19 - immutable int column; ///< column: 1, 2, ... 18 + immutable string filename; /// name of the source file 19 + immutable int lineno; /// line number, 1, 2, ... 20 + immutable int column; /// column, 1, 2, ... 20 21 21 22 override string toString() const 22 23 { return sprintf!"%s:%d:%d"(filename, lineno, column); } 23 24 24 25 mixin SimpleConstructor; 25 26 mixin SimpleCompare; 26 27 } ................................................................................ 44 45 } 45 46 46 47 /// Represents a lexer token 47 48 48 49 class Token 49 50 { 50 51 enum Kind {identifier, stringLiteral, number}; 51 - immutable LexPosition pos; ///< position where the token occurred in the source 52 - immutable string str; ///< the token string itself 53 - immutable Kind kind; ///< which kind of token? 52 + immutable LexPosition pos; /// position where the token occurred in the source 53 + immutable string str; /// the token string itself 54 + immutable Kind kind; /// which kind of token? 54 55 55 56 mixin SimpleConstructor; 56 57 mixin SimpleCompare; 57 58 } 58 59 59 60 unittest 60 61 { ................................................................................ 85 86 return new Lexer(str, filename, lineno, column); 86 87 } 87 88 88 89 /// Lexer is a forward range of Tokens 89 90 90 91 class Lexer 91 92 { 93 + /// Range primitive 92 94 bool empty() /*@property*/ 93 95 { 94 96 return current is null; 95 97 } 96 98 99 + /// Range primitive 97 100 Token front() /*@property*/ 98 101 { 99 102 return std.exception.enforce(current, "Lexer has already reached the end"); 100 103 } 101 104 105 + /// Range primitive 102 106 void popFront() /*@property*/ 103 107 { 104 108 std.exception.enforce(current, "Lexer has already reached the end"); 105 109 current = readNext(); 106 110 } 107 111 112 + /// Range primitive 108 113 Lexer save() /*@property*/ 109 114 { 110 115 return new Lexer(buffer, filename, lineno, column, current); 111 116 } 112 117 113 118 private: // implementation 114 119 ................................................................................ 326 331 assert( lex3.front.kind == Token.Kind.number ); 327 332 } 328 333 329 334 unittest 330 335 { 331 336 //!! be sure to run the unittest on the root of the source directory 332 337 auto lexf = lexerFromFile("polemy/lex.d"); 338 + lexf = find!`a.str == "module"`(lexf); 333 339 assert( lexf.front.str == "module", lexf.front.str ); 334 340 assert( lexf.front.pos.filename == "polemy/lex.d" ); 335 - assert( lexf.front.pos.lineno == 1 ); 341 + assert( lexf.front.pos.lineno == 7 ); 336 342 assert( lexf.front.pos.column == 1 ); 337 343 lexf.popFront; 338 344 assert( lexf.front.str == "polemy" ); 339 - assert( lexf.front.pos.lineno == 1 ); 345 + assert( lexf.front.pos.lineno == 7 ); 340 346 assert( lexf.front.pos.column == 8 ); 341 347 lexf.popFront; 342 348 assert( lexf.front.str == "." ); 343 349 lexf.popFront; 344 350 assert( lexf.front.str == "lex" ); 345 351 lexf.popFront; 346 352 assert( lexf.front.str == ";" ); 347 353 lexf.popFront; 348 354 assert( lexf.front.str == "import" ); 349 - assert( lexf.front.pos.lineno == 2 ); 355 + assert( lexf.front.pos.lineno == 8 ); 350 356 assert( lexf.front.pos.column == 1 ); 351 357 } 352 358 353 359 unittest 354 360 { 355 361 auto lex = lexerFromString(`my # comment should 356 362 # hey!!