> 1 /** > 2 * Authors: k.inaba > 3 * License: NYSL 0.9982 http://www.kmonos.net/nysl/ > 4 * > 5 * Lexer for Polemy programming language. > 6 */ 1 module polemy.lex; 7 module polemy.lex; 2 import polemy._common; 8 import polemy._common; 3 /* < 4 * Author: k.inaba < 5 * License: NYSL 0.9982 (http://www.kmonos.net/nysl/ < 6 * Lexer for the polemy programming language < 7 */ < 8 9 9 import std.file : readText; 10 import std.file : readText; 10 import std.string : munch; 11 import std.string : munch; 11 import std.ctype; 12 import std.ctype; 12 13 13 /// Represents a position in a source code 14 /// Represents a position in a source code 14 15 15 class LexPosition 16 class LexPosition 16 { 17 { 17 immutable string filename; ///< name of the source file | 18 immutable string filename; /// name of the source file 18 immutable int lineno; ///< line number: 1, 2, ... | 19 immutable int lineno; /// line number, 1, 2, ... 19 immutable int column; ///< column: 1, 2, ... | 20 immutable int column; /// column, 1, 2, ... 20 21 21 override string toString() const 22 override string toString() const 22 { return sprintf!"%s:%d:%d"(filename, lineno, column); } 23 { return sprintf!"%s:%d:%d"(filename, lineno, column); } 23 24 24 mixin SimpleConstructor; 25 mixin SimpleConstructor; 25 mixin SimpleCompare; 26 mixin SimpleCompare; 26 } 27 } ................................................................................................................................................................................ 44 } 45 } 45 46 46 /// Represents a lexer token 47 /// Represents a lexer token 47 48 48 class Token 49 class Token 49 { 50 { 50 enum Kind {identifier, stringLiteral, number}; 51 enum Kind {identifier, stringLiteral, number}; 51 immutable LexPosition pos; ///< position where the token occurred in th | 52 immutable LexPosition pos; /// position where the token occurred in the 52 immutable string str; ///< the token string itself | 53 immutable string str; /// the token string itself 53 immutable Kind kind; ///< which kind of token? | 54 immutable Kind kind; /// which kind of token? 54 55 55 mixin SimpleConstructor; 56 mixin SimpleConstructor; 56 mixin SimpleCompare; 57 mixin SimpleCompare; 57 } 58 } 58 59 59 unittest 60 unittest 60 { 61 { ................................................................................................................................................................................ 85 return new Lexer(str, filename, lineno, column); 86 return new Lexer(str, filename, lineno, column); 86 } 87 } 87 88 88 /// Lexer is a forward range of Tokens 89 /// Lexer is a forward range of Tokens 89 90 90 class Lexer 91 class Lexer 91 { 92 { > 93 /// Range primitive 92 bool empty() /*@property*/ 94 bool empty() /*@property*/ 93 { 95 { 94 return current is null; 96 return current is null; 95 } 97 } 96 98 > 99 /// Range primitive 97 Token front() /*@property*/ 100 Token front() /*@property*/ 98 { 101 { 99 return std.exception.enforce(current, "Lexer has already reached 102 return std.exception.enforce(current, "Lexer has already reached 100 } 103 } 101 104 > 105 /// Range primitive 102 void popFront() /*@property*/ 106 void popFront() /*@property*/ 103 { 107 { 104 std.exception.enforce(current, "Lexer has already reached the en 108 std.exception.enforce(current, "Lexer has already reached the en 105 current = readNext(); 109 current = readNext(); 106 } 110 } 107 111 > 112 /// Range primitive 108 Lexer save() /*@property*/ 113 Lexer save() /*@property*/ 109 { 114 { 110 return new Lexer(buffer, filename, lineno, column, current); 115 return new Lexer(buffer, filename, lineno, column, current); 111 } 116 } 112 117 113 private: // implementation 118 private: // implementation 114 119 ................................................................................................................................................................................ 326 assert( lex3.front.kind == Token.Kind.number ); 331 assert( lex3.front.kind == Token.Kind.number ); 327 } 332 } 328 333 329 unittest 334 unittest 330 { 335 { 331 //!! be sure to run the unittest on the root of the source directory 336 //!! be sure to run the unittest on the root of the source directory 332 auto lexf = lexerFromFile("polemy/lex.d"); 337 auto lexf = lexerFromFile("polemy/lex.d"); > 338 lexf = find!`a.str == "module"`(lexf); 333 assert( lexf.front.str == "module", lexf.front.str ); 339 assert( lexf.front.str == "module", lexf.front.str ); 334 assert( lexf.front.pos.filename == "polemy/lex.d" ); 340 assert( lexf.front.pos.filename == "polemy/lex.d" ); 335 assert( lexf.front.pos.lineno == 1 ); | 341 assert( lexf.front.pos.lineno == 7 ); 336 assert( lexf.front.pos.column == 1 ); 342 assert( lexf.front.pos.column == 1 ); 337 lexf.popFront; 343 lexf.popFront; 338 assert( lexf.front.str == "polemy" ); 344 assert( lexf.front.str == "polemy" ); 339 assert( lexf.front.pos.lineno == 1 ); | 345 assert( lexf.front.pos.lineno == 7 ); 340 assert( lexf.front.pos.column == 8 ); 346 assert( lexf.front.pos.column == 8 ); 341 lexf.popFront; 347 lexf.popFront; 342 assert( lexf.front.str == "." ); 348 assert( lexf.front.str == "." ); 343 lexf.popFront; 349 lexf.popFront; 344 assert( lexf.front.str == "lex" ); 350 assert( lexf.front.str == "lex" ); 345 lexf.popFront; 351 lexf.popFront; 346 assert( lexf.front.str == ";" ); 352 assert( lexf.front.str == ";" ); 347 lexf.popFront; 353 lexf.popFront; 348 assert( lexf.front.str == "import" ); 354 assert( lexf.front.str == "import" ); 349 assert( lexf.front.pos.lineno == 2 ); | 355 assert( lexf.front.pos.lineno == 8 ); 350 assert( lexf.front.pos.column == 1 ); 356 assert( lexf.front.pos.column == 1 ); 351 } 357 } 352 358 353 unittest 359 unittest 354 { 360 { 355 auto lex = lexerFromString(`my # comment should 361 auto lex = lexerFromString(`my # comment should 356 # hey!! 362 # hey!!