4198578702 2010-11-07        kinaba: /**
4198578702 2010-11-07        kinaba:  * Authors: k.inaba
4198578702 2010-11-07        kinaba:  * License: NYSL 0.9982 http://www.kmonos.net/nysl/
4198578702 2010-11-07        kinaba:  *
4198578702 2010-11-07        kinaba:  * Lexer for Polemy programming language.
4198578702 2010-11-07        kinaba:  */
423f308350 2010-11-07        kinaba: module polemy.lex;
423f308350 2010-11-07        kinaba: import polemy._common;
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: import std.file : readText;
423f308350 2010-11-07        kinaba: import std.string : munch;
423f308350 2010-11-07        kinaba: import std.ctype;
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: /// Represents a position in a source code
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: class LexPosition
423f308350 2010-11-07        kinaba: {
4198578702 2010-11-07        kinaba: 	immutable string filename; /// name of the source file
4198578702 2010-11-07        kinaba: 	immutable int    lineno;   /// line number, 1, 2, ...
4198578702 2010-11-07        kinaba: 	immutable int    column;   /// column, 1, 2, ...
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	override string toString() const
423f308350 2010-11-07        kinaba: 		{ return sprintf!"%s:%d:%d"(filename, lineno, column); }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	mixin SimpleConstructor;
423f308350 2010-11-07        kinaba: 	mixin SimpleCompare;
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: unittest
423f308350 2010-11-07        kinaba: {
423f308350 2010-11-07        kinaba: 	auto p = new LexPosition("hello.cpp", 123, 45);
423f308350 2010-11-07        kinaba: 	auto q = new LexPosition("hello.cpp", 123, 46);
423f308350 2010-11-07        kinaba: 
61998c472a 2010-11-08        kinaba: 	assert_eq( p.filename, "hello.cpp" );
61998c472a 2010-11-08        kinaba: 	assert_eq( p.lineno, 123 );
61998c472a 2010-11-08        kinaba: 	assert_eq( p.column, 45 );
61998c472a 2010-11-08        kinaba: 	assert_eq( to!string(p), "hello.cpp:123:45" );
61998c472a 2010-11-08        kinaba: 	assert_lt( p, q );
61998c472a 2010-11-08        kinaba: 	assert_ne( p, q );
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	assert( !__traits(compiles, new LexPosition) );
423f308350 2010-11-07        kinaba: 	assert( !__traits(compiles, p.filename="foo") );
423f308350 2010-11-07        kinaba: 	assert( !__traits(compiles, p.lineno  =789) );
423f308350 2010-11-07        kinaba: 	assert( !__traits(compiles, p.column  =222) );
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: /// Represents a lexer token
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: class Token
423f308350 2010-11-07        kinaba: {
8d297342aa 2010-11-08        kinaba: 	immutable LexPosition pos;    /// Position where the token occurred in the source
8d297342aa 2010-11-08        kinaba: 	immutable string      str;    /// The token string itself
8d297342aa 2010-11-08        kinaba: 	immutable bool        quoted; /// Was it a "quoted" token or unquoted?
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	mixin SimpleConstructor;
423f308350 2010-11-07        kinaba: 	mixin SimpleCompare;
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: unittest
423f308350 2010-11-07        kinaba: {
423f308350 2010-11-07        kinaba: 	auto p = new immutable(LexPosition)("hello.cpp", 123, 45);
8d297342aa 2010-11-08        kinaba: 	auto t = new Token(p, "class", false);
8d297342aa 2010-11-08        kinaba: 	auto u = new Token(p, "class", true);
423f308350 2010-11-07        kinaba: 
61998c472a 2010-11-08        kinaba: 	assert_eq( t.pos, p );
61998c472a 2010-11-08        kinaba: 	assert_eq( t.str, "class" );
8d297342aa 2010-11-08        kinaba: 	assert( !t.quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( t, new Token(p, "class", false) );
8d297342aa 2010-11-08        kinaba: 	assert_lt( t, new Token(p, "struct", false) );
8d297342aa 2010-11-08        kinaba: 	assert_ne( t, u );
8d297342aa 2010-11-08        kinaba: 	assert( u.quoted );
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	assert( !__traits(compiles, new Token) );
423f308350 2010-11-07        kinaba: 	assert( !__traits(compiles, t.pos=p) );
423f308350 2010-11-07        kinaba: 	assert( !__traits(compiles, t.str=789) );
8d297342aa 2010-11-08        kinaba: 	assert( !__traits(compiles, t.quoted=true) );
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: /// Named Construtor for Lexer
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: Lexer lexerFromFile(T...)( string filename, T rest )
423f308350 2010-11-07        kinaba: {
423f308350 2010-11-07        kinaba: 	return lexerFromString( std.file.readText(filename), filename, rest );
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: /// Named Construtor for Lexer
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: Lexer lexerFromString( string str, string filename="<unnamed>", int lineno=1, int column=1 )
423f308350 2010-11-07        kinaba: {
423f308350 2010-11-07        kinaba: 	return new Lexer(str, filename, lineno, column);
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: /// Lexer is a forward range of Tokens
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: class Lexer
423f308350 2010-11-07        kinaba: {
4198578702 2010-11-07        kinaba: 	/// Range primitive
423f308350 2010-11-07        kinaba: 	bool empty() /*@property*/
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		return current is null;
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
4198578702 2010-11-07        kinaba: 	/// Range primitive
423f308350 2010-11-07        kinaba: 	Token front() /*@property*/
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		return std.exception.enforce(current, "Lexer has already reached the end");
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
4198578702 2010-11-07        kinaba: 	/// Range primitive
423f308350 2010-11-07        kinaba: 	void popFront() /*@property*/
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		std.exception.enforce(current, "Lexer has already reached the end");
423f308350 2010-11-07        kinaba: 		current = readNext();
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
4198578702 2010-11-07        kinaba: 	/// Range primitive
423f308350 2010-11-07        kinaba: 	Lexer save() /*@property*/
423f308350 2010-11-07        kinaba: 	{
8d297342aa 2010-11-08        kinaba: 		return new Lexer(this.tupleof);
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: private: // implementation
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	string buffer;
423f308350 2010-11-07        kinaba: 	string filename;
423f308350 2010-11-07        kinaba: 	int    lineno;
423f308350 2010-11-07        kinaba: 	int    column;
423f308350 2010-11-07        kinaba: 	Token  current;
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	invariant()
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		assert( buffer.empty || !std.ctype.isspace(buffer[0]) );
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	this( string buffer, string filename, int lineno, int column, Token current=null )
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		this.buffer   = buffer;
423f308350 2010-11-07        kinaba: 		this.filename = filename;
423f308350 2010-11-07        kinaba: 		this.lineno   = lineno;
423f308350 2010-11-07        kinaba: 		this.column   = column;
423f308350 2010-11-07        kinaba: 		skipws();
423f308350 2010-11-07        kinaba: 		this.current  = (current is null ? readNext() : current);
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	void skipws()
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		bool progress = false;
423f308350 2010-11-07        kinaba: 		do
423f308350 2010-11-07        kinaba: 		{
423f308350 2010-11-07        kinaba: 			string ws = buffer.munch(" \t");
423f308350 2010-11-07        kinaba: 			column += ws.length;
423f308350 2010-11-07        kinaba: 			progress = !ws.empty;
423f308350 2010-11-07        kinaba: 			while( !buffer.empty && (buffer[0]=='\r' || buffer[0]=='\n') )
423f308350 2010-11-07        kinaba: 			{
423f308350 2010-11-07        kinaba: 				progress = true;
423f308350 2010-11-07        kinaba: 				if( buffer[0] == '\n' )
423f308350 2010-11-07        kinaba: 					buffer = buffer[1..$];
423f308350 2010-11-07        kinaba: 				else // if( buffer.front == '\r' )
423f308350 2010-11-07        kinaba: 				{
423f308350 2010-11-07        kinaba: 					buffer = buffer[1..$];
423f308350 2010-11-07        kinaba: 					if( !buffer.empty && buffer[0]=='\n' )
423f308350 2010-11-07        kinaba: 						buffer = buffer[1..$];
423f308350 2010-11-07        kinaba: 				}
423f308350 2010-11-07        kinaba: 				lineno ++;
423f308350 2010-11-07        kinaba: 				column = 1;
423f308350 2010-11-07        kinaba: 			}
423f308350 2010-11-07        kinaba: 		}while( progress );
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	char readChar()
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		scope(exit) {
423f308350 2010-11-07        kinaba: 			buffer = buffer[1..$];
423f308350 2010-11-07        kinaba: 			column ++;
423f308350 2010-11-07        kinaba: 		}
423f308350 2010-11-07        kinaba: 		return buffer[0];
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	/// This is the main lexing routine
423f308350 2010-11-07        kinaba: 	Token readNext()
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		if( buffer.empty )
423f308350 2010-11-07        kinaba: 			return null;
423f308350 2010-11-07        kinaba: 		scope(exit)
423f308350 2010-11-07        kinaba: 			skipws();
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 		if( isSymbol(buffer[0]) )
423f308350 2010-11-07        kinaba: 		{
423f308350 2010-11-07        kinaba: 			if( buffer[0] == '#' )
423f308350 2010-11-07        kinaba: 			{
423f308350 2010-11-07        kinaba: 				// skip comment
423f308350 2010-11-07        kinaba: 				while( !buffer.empty && (buffer[0]!='\n' && buffer[0]!='\r') )
423f308350 2010-11-07        kinaba: 					readChar();
423f308350 2010-11-07        kinaba: 				skipws();
423f308350 2010-11-07        kinaba: 				return readNext();
423f308350 2010-11-07        kinaba: 			}
423f308350 2010-11-07        kinaba: 			else if( buffer[0] == '"' )
423f308350 2010-11-07        kinaba: 			{
423f308350 2010-11-07        kinaba: 				// string literal
423f308350 2010-11-07        kinaba: 				auto pos = currentPosition();
423f308350 2010-11-07        kinaba: 				string lit;
423f308350 2010-11-07        kinaba: 				readChar();
423f308350 2010-11-07        kinaba: 				while( !buffer.empty && buffer[0]!='"' )
423f308350 2010-11-07        kinaba: 				{
423f308350 2010-11-07        kinaba: 					// read one char
423f308350 2010-11-07        kinaba: 					char c = readChar();
423f308350 2010-11-07        kinaba: 					if( c == '\\' )
423f308350 2010-11-07        kinaba: 					{
423f308350 2010-11-07        kinaba: 						if( !buffer.empty && (buffer[0]=='\\' || buffer[0]=='"') )
423f308350 2010-11-07        kinaba: 							lit ~= readChar();
423f308350 2010-11-07        kinaba: 						else
423f308350 2010-11-07        kinaba: 							lit ~= c;
423f308350 2010-11-07        kinaba: 					}
423f308350 2010-11-07        kinaba: 					else if( c == '\n' )
423f308350 2010-11-07        kinaba: 					{
423f308350 2010-11-07        kinaba: 						lit ~= c;
423f308350 2010-11-07        kinaba: 						lineno++;
423f308350 2010-11-07        kinaba: 						column = 1;
423f308350 2010-11-07        kinaba: 					}
423f308350 2010-11-07        kinaba: 					else if( c == '\r' )
423f308350 2010-11-07        kinaba: 					{
423f308350 2010-11-07        kinaba: 						if( !buffer.empty && buffer[0]=='\n' )
423f308350 2010-11-07        kinaba: 							readChar();
423f308350 2010-11-07        kinaba: 						lit ~= '\n';
423f308350 2010-11-07        kinaba: 						lineno++;
423f308350 2010-11-07        kinaba: 						column = 1;
423f308350 2010-11-07        kinaba: 					}
423f308350 2010-11-07        kinaba: 					else
423f308350 2010-11-07        kinaba: 						lit ~= c;
423f308350 2010-11-07        kinaba: 				}
423f308350 2010-11-07        kinaba: 				if( !buffer.empty )
423f308350 2010-11-07        kinaba: 					readChar();
8d297342aa 2010-11-08        kinaba: 				return new Token(pos, lit, true);
423f308350 2010-11-07        kinaba: 			}
423f308350 2010-11-07        kinaba: 			else
423f308350 2010-11-07        kinaba: 			{
423f308350 2010-11-07        kinaba: 				// normal symbol
423f308350 2010-11-07        kinaba: 				auto pos = currentPosition();
423f308350 2010-11-07        kinaba: 				auto str = ""~readChar();
8d297342aa 2010-11-08        kinaba: 				return new Token(pos, str, false);
423f308350 2010-11-07        kinaba: 			}
423f308350 2010-11-07        kinaba: 		}
423f308350 2010-11-07        kinaba: 		else
423f308350 2010-11-07        kinaba: 		{
423f308350 2010-11-07        kinaba: 			auto pos = currentPosition();
423f308350 2010-11-07        kinaba: 			int i = 0;
423f308350 2010-11-07        kinaba: 			while( i<buffer.length && !std.ctype.isspace(buffer[i]) && !isSymbol(buffer[i]) )
423f308350 2010-11-07        kinaba: 				++i;
423f308350 2010-11-07        kinaba: 			auto str = buffer[0 .. i];
423f308350 2010-11-07        kinaba: 			buffer   = buffer[i .. $];
423f308350 2010-11-07        kinaba: 			column  += i;
8d297342aa 2010-11-08        kinaba: 			return new Token(pos, str, false);
423f308350 2010-11-07        kinaba: 		}
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	bool isSymbol(char c)
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		return (0x21<=c && c<=0x7f && !std.ctype.isalnum(c) && c!='_');
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	immutable(LexPosition) currentPosition()
423f308350 2010-11-07        kinaba: 	{
423f308350 2010-11-07        kinaba: 		return new immutable(LexPosition)(filename, lineno, column);
423f308350 2010-11-07        kinaba: 	}
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: unittest
423f308350 2010-11-07        kinaba: {
423f308350 2010-11-07        kinaba: 	assert( std.range.isForwardRange!(Lexer) );
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: unittest
423f308350 2010-11-07        kinaba: {
8d297342aa 2010-11-08        kinaba: 	auto lex = lexerFromString("this	is a \t\r\n pen :-(   ");
423f308350 2010-11-07        kinaba: 	Token[] ts = std.array.array(lex);
423f308350 2010-11-07        kinaba: 
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[0].pos.lineno, 1 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[0].pos.column, 1 );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[0].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[0].str, "this" );
8d297342aa 2010-11-08        kinaba: 
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[1].pos.lineno, 1 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[1].pos.column, 6 );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[1].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[1].str, "is" );
423f308350 2010-11-07        kinaba: 
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[2].pos.lineno, 1 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[2].pos.column, 9 );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[2].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[2].str, "a" );
423f308350 2010-11-07        kinaba: 
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[3].pos.lineno, 2 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[3].pos.column, 2 );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[3].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[3].str, "pen" );
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: 	// consecutive symbols are always separated
423f308350 2010-11-07        kinaba: 	// hence, no "++" or "<<" or ...
423f308350 2010-11-07        kinaba: 
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[4].pos.lineno, 2 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[4].pos.column, 6 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[4].str, ":" );
8d297342aa 2010-11-08        kinaba: 
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[5].pos.lineno, 2 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[5].pos.column, 7 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[5].str, "-" );
423f308350 2010-11-07        kinaba: 
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[6].pos.lineno, 2 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[6].pos.column, 8 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[6].str, "(" );
423f308350 2010-11-07        kinaba: 
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts.length, 7 );
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: unittest
423f308350 2010-11-07        kinaba: {
423f308350 2010-11-07        kinaba: 	auto lex2 = lexerFromString(" a12\n3a 5 ");
8d297342aa 2010-11-08        kinaba: 	assert_eq( lex2.front.str, "a12" );
423f308350 2010-11-07        kinaba: 	lex2.popFront;
423f308350 2010-11-07        kinaba: 	auto lex3 = lex2.save;
8d297342aa 2010-11-08        kinaba: 	assert_eq( lex2.front.str, "3a" );
423f308350 2010-11-07        kinaba: 	lex2.popFront;
8d297342aa 2010-11-08        kinaba: 	assert_eq( lex3.front.str, "3a" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( lex2.front.str, "5" );
423f308350 2010-11-07        kinaba: 	lex2.popFront;
423f308350 2010-11-07        kinaba: 	lex3.popFront;
423f308350 2010-11-07        kinaba: 	assert( lex2.empty );
423f308350 2010-11-07        kinaba: 	assert( !lex3.empty );
8d297342aa 2010-11-08        kinaba: 	assert_eq( lex3.front.str, "5" );
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: unittest
423f308350 2010-11-07        kinaba: {
423f308350 2010-11-07        kinaba: //!! be sure to run the unittest on the root of the source directory
423f308350 2010-11-07        kinaba: 	auto lexf = lexerFromFile("polemy/lex.d");
4198578702 2010-11-07        kinaba: 	lexf = find!`a.str == "module"`(lexf);
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.str, "module" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.pos.filename, "polemy/lex.d" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.pos.lineno, 7 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.pos.column, 1 );
8d297342aa 2010-11-08        kinaba: 	lexf.popFront;
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.str, "polemy" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.pos.lineno, 7 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.pos.column, 8 );
8d297342aa 2010-11-08        kinaba: 	lexf.popFront;
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.str, "." );
423f308350 2010-11-07        kinaba: 	lexf.popFront;
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.str, "lex" );
423f308350 2010-11-07        kinaba: 	lexf.popFront;
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.str, ";" );
423f308350 2010-11-07        kinaba: 	lexf.popFront;
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.str, "import" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.pos.lineno, 8 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( lexf.front.pos.column, 1 );
423f308350 2010-11-07        kinaba: }
423f308350 2010-11-07        kinaba: 
423f308350 2010-11-07        kinaba: unittest
423f308350 2010-11-07        kinaba: {
8d297342aa 2010-11-08        kinaba: 	auto lex = lexerFromString(`my # comment should`~"\r\n"~`# hey!!
423f308350 2010-11-07        kinaba: be ignored.
423f308350 2010-11-07        kinaba: hahaha"hihihi""hu\\\"huhu"#123 aa
8d297342aa 2010-11-08        kinaba: 123 aa "aaa`~"\r\n"~`bbb # 123`~"\r\n"~`eee"
423f308350 2010-11-07        kinaba: zzz
423f308350 2010-11-07        kinaba: `);
423f308350 2010-11-07        kinaba: 	Token[] ts = std.array.array(lex);
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[0].str, "my" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[0].pos.lineno, 1 );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[0].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[1].str, "be" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[1].pos.lineno, 3 );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[1].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[2].str, "ignored" );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[2].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[3].str, "." );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[3].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[4].str, "hahaha" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[4].pos.lineno, 4 );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[4].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[5].str, "hihihi" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[5].pos.lineno, 4 );
8d297342aa 2010-11-08        kinaba: 	assert(    ts[5].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[6].str, `hu\"huhu` );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[6].pos.lineno, 4 );
8d297342aa 2010-11-08        kinaba: 	assert(    ts[6].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[7].str, "123" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[7].pos.lineno, 5 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[8].str, "aa" );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[9].pos.lineno, 5 );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[9].str, "aaa\nbbb # 123\neee" );
8d297342aa 2010-11-08        kinaba: 	assert(    ts[9].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts[10].pos.lineno, 8 );
8d297342aa 2010-11-08        kinaba: 	assert(   !ts[10].quoted );
8d297342aa 2010-11-08        kinaba: 	assert_eq( ts.length, 11 );
423f308350 2010-11-07        kinaba: }