Now, lexer can deal with character literals with white spaces, hurray!
parent
78c4572f5e
commit
67b2379d03
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
Lexer::Lexer()
|
Lexer::Lexer()
|
||||||
{
|
{
|
||||||
string keys[] = {"function", "class", "penis"};
|
string keys[] = {"function", "class", "for", "while", "if", "else"};
|
||||||
keywords.assign(keys, keys+3);
|
keywords.assign(keys, keys+6);
|
||||||
|
|
||||||
char punct[] = {'.', ',', ';', '{', '}', '[', ']', '(', ')'};
|
char punct[] = {'.', ',', ';', '{', '}', '[', ']', '(', ')'};
|
||||||
punctuation.assign(punct, punct+9);
|
punctuation.assign(punct, punct+9);
|
||||||
|
@ -62,6 +62,7 @@ pair<string, Token> Lexer::parse_token(string line)
|
||||||
{
|
{
|
||||||
string token_value = "";
|
string token_value = "";
|
||||||
unsigned int i;
|
unsigned int i;
|
||||||
|
bool in_char_literal = false;
|
||||||
for(i=0; i< line.size(); i++)
|
for(i=0; i< line.size(); i++)
|
||||||
{
|
{
|
||||||
if(token_value == "" && isspace(line[i])) continue;
|
if(token_value == "" && isspace(line[i])) continue;
|
||||||
|
@ -69,6 +70,14 @@ pair<string, Token> Lexer::parse_token(string line)
|
||||||
if(isalnum(line[i]) || line[i] == '\"' || line[i]== '_')
|
if(isalnum(line[i]) || line[i] == '\"' || line[i]== '_')
|
||||||
{
|
{
|
||||||
token_value += line[i];
|
token_value += line[i];
|
||||||
|
if(line[i] == '\"')
|
||||||
|
{
|
||||||
|
if(in_char_literal)
|
||||||
|
{
|
||||||
|
in_char_literal = false;
|
||||||
|
}
|
||||||
|
else in_char_literal = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else if(ispunct(line[i]))
|
else if(ispunct(line[i]))
|
||||||
{
|
{
|
||||||
|
@ -88,6 +97,10 @@ pair<string, Token> Lexer::parse_token(string line)
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
else if (in_char_literal && isspace(line[i]))
|
||||||
|
{
|
||||||
|
token_value += line[i];
|
||||||
|
}
|
||||||
else break;
|
else break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -14,13 +14,7 @@ class Lexer
|
||||||
Lexer();
|
Lexer();
|
||||||
virtual ~Lexer();
|
virtual ~Lexer();
|
||||||
|
|
||||||
bool is_keyword(string value);
|
|
||||||
bool is_punctuation(char c);
|
|
||||||
bool is_operator(string value );
|
|
||||||
|
|
||||||
vector<string> keywords;
|
|
||||||
vector<char> punctuation;
|
|
||||||
vector<string> operators;
|
|
||||||
|
|
||||||
void add_keyword(string word);
|
void add_keyword(string word);
|
||||||
void add_punctuation_char(char c);
|
void add_punctuation_char(char c);
|
||||||
|
@ -32,6 +26,14 @@ class Lexer
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
private:
|
private:
|
||||||
|
|
||||||
|
bool is_keyword(string value);
|
||||||
|
bool is_punctuation(char c);
|
||||||
|
bool is_operator(string value );
|
||||||
|
|
||||||
|
vector<string> keywords;
|
||||||
|
vector<char> punctuation;
|
||||||
|
vector<string> operators;
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif // LEXER_H
|
#endif // LEXER_H
|
||||||
|
|
|
@ -3,7 +3,6 @@
|
||||||
|
|
||||||
Parser::Parser(vector<Token> tokens)
|
Parser::Parser(vector<Token> tokens)
|
||||||
{
|
{
|
||||||
//token_stream.push_back(Token(t_symbol, "dupa"));
|
|
||||||
error_message = "***ERRORS DURING PARSING***\n";
|
error_message = "***ERRORS DURING PARSING***\n";
|
||||||
report_message = "***PARSER REPORT***\n";
|
report_message = "***PARSER REPORT***\n";
|
||||||
token_stream = tokens;
|
token_stream = tokens;
|
||||||
|
@ -140,7 +139,6 @@ void Parser::statement()
|
||||||
{
|
{
|
||||||
if(accept("{"))
|
if(accept("{"))
|
||||||
{
|
{
|
||||||
report("GO Deeper\n");
|
|
||||||
while(!accept("}"))
|
while(!accept("}"))
|
||||||
{
|
{
|
||||||
statement();
|
statement();
|
||||||
|
|
|
@ -10,23 +10,26 @@ class Parser
|
||||||
public:
|
public:
|
||||||
Parser(vector<Token> tokens);
|
Parser(vector<Token> tokens);
|
||||||
virtual ~Parser();
|
virtual ~Parser();
|
||||||
|
void interpret();
|
||||||
|
string report_message;
|
||||||
|
string error_message;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
private:
|
||||||
Token current_token;
|
Token current_token;
|
||||||
string tok_value;
|
string tok_value;
|
||||||
vector<Token> token_stream;
|
vector<Token> token_stream;
|
||||||
int position_in_stream;
|
int position_in_stream;
|
||||||
|
|
||||||
string report_message;
|
|
||||||
string error_message;
|
|
||||||
|
|
||||||
bool read_next();
|
bool read_next();
|
||||||
bool peek(string s);
|
bool peek(string s);
|
||||||
bool accept(string s);
|
bool accept(string s);
|
||||||
bool expect(string s);
|
bool expect(string s);
|
||||||
|
bool is_type();
|
||||||
|
|
||||||
void error(string s);
|
void error(string s);
|
||||||
void report(string s);
|
void report(string s);
|
||||||
void interpret();
|
|
||||||
void statement();
|
void statement();
|
||||||
|
|
||||||
void add_expr();
|
void add_expr();
|
||||||
|
@ -35,10 +38,6 @@ class Parser
|
||||||
void rel_expr();
|
void rel_expr();
|
||||||
void eq_expr();
|
void eq_expr();
|
||||||
void expr();
|
void expr();
|
||||||
|
|
||||||
bool is_type();
|
|
||||||
protected:
|
|
||||||
private:
|
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif // PARSER_H
|
#endif // PARSER_H
|
||||||
|
|
Binary file not shown.
|
@ -8,7 +8,7 @@ using namespace std;
|
||||||
|
|
||||||
void test_lexer()
|
void test_lexer()
|
||||||
{
|
{
|
||||||
string test_line = "def i; bulb; i + 3; string banan; banan = \"banan\"; string kaboom(num how_many_times) { def z; }";
|
string test_line = "def i; bulb; i + 3; string banan; banan = \"banan and other stuff\"; string kaboom(num how_many_times) { def z; }";
|
||||||
string test_line2 = "def how_many_trees = 1; how_many_trees + 3 == 2; num cut_tree( num how_many) {return how_many -1; != <=}";
|
string test_line2 = "def how_many_trees = 1; how_many_trees + 3 == 2; num cut_tree( num how_many) {return how_many -1; != <=}";
|
||||||
Lexer lexer;
|
Lexer lexer;
|
||||||
vector<Token> tokens = lexer.parse_line(test_line);
|
vector<Token> tokens = lexer.parse_line(test_line);
|
||||||
|
@ -22,10 +22,10 @@ void test_lexer()
|
||||||
|
|
||||||
tokens = lexer.parse_line(test_line2);
|
tokens = lexer.parse_line(test_line2);
|
||||||
|
|
||||||
for(int i=0; i< tokens.size(); i++)
|
/*for(int i=0; i< tokens.size(); i++)
|
||||||
{
|
{
|
||||||
cout << tokens[i].get_value() << " type: " << tokens[i].get_type() << endl;
|
cout << tokens[i].get_value() << " type: " << tokens[i].get_type() << endl;
|
||||||
}
|
}*/
|
||||||
}
|
}
|
||||||
|
|
||||||
void test_parser()
|
void test_parser()
|
||||||
|
@ -52,7 +52,7 @@ void test_parser()
|
||||||
int main()
|
int main()
|
||||||
{
|
{
|
||||||
cout << "Hello world!" << endl;
|
cout << "Hello world!" << endl;
|
||||||
test_parser();
|
//test_parser();
|
||||||
//test_lexer();
|
test_lexer();
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -34,6 +34,8 @@
|
||||||
</Compiler>
|
</Compiler>
|
||||||
<Unit filename="Lexer.cpp" />
|
<Unit filename="Lexer.cpp" />
|
||||||
<Unit filename="Lexer.h" />
|
<Unit filename="Lexer.h" />
|
||||||
|
<Unit filename="Parser.cpp" />
|
||||||
|
<Unit filename="Parser.h" />
|
||||||
<Unit filename="Token.cpp" />
|
<Unit filename="Token.cpp" />
|
||||||
<Unit filename="Token.h" />
|
<Unit filename="Token.h" />
|
||||||
<Unit filename="main.cpp" />
|
<Unit filename="main.cpp" />
|
||||||
|
|
|
@ -5,10 +5,10 @@
|
||||||
1351424620 /home/attero/development/sencha-lang/sencha/Token.h
|
1351424620 /home/attero/development/sencha-lang/sencha/Token.h
|
||||||
<string>
|
<string>
|
||||||
|
|
||||||
1351428918 source:/home/attero/development/sencha-lang/sencha/Lexer.cpp
|
1351540082 source:/home/attero/development/sencha-lang/sencha/Lexer.cpp
|
||||||
"Lexer.h"
|
"Lexer.h"
|
||||||
|
|
||||||
1351413156 /home/attero/development/sencha-lang/sencha/Lexer.h
|
1351539503 /home/attero/development/sencha-lang/sencha/Lexer.h
|
||||||
<vector>
|
<vector>
|
||||||
<string>
|
<string>
|
||||||
<utility>
|
<utility>
|
||||||
|
@ -16,19 +16,24 @@
|
||||||
<iostream>
|
<iostream>
|
||||||
"Token.h"
|
"Token.h"
|
||||||
|
|
||||||
1351429258 source:/home/attero/development/sencha-lang/sencha/main.cpp
|
1351539590 source:/home/attero/development/sencha-lang/sencha/main.cpp
|
||||||
<iostream>
|
<iostream>
|
||||||
<string>
|
<string>
|
||||||
"Token.h"
|
"Token.h"
|
||||||
"Lexer.h"
|
"Lexer.h"
|
||||||
"Parser.h"
|
"Parser.h"
|
||||||
|
|
||||||
1351429380 source:/home/attero/development/sencha-lang/sencha/Parser.cpp
|
1351539503 source:/home/attero/development/sencha-lang/sencha/Parser.cpp
|
||||||
"Parser.h"
|
"Parser.h"
|
||||||
"iostream"
|
"iostream"
|
||||||
|
|
||||||
1351424740 /home/attero/development/sencha-lang/sencha/Parser.h
|
1351539525 /home/attero/development/sencha-lang/sencha/Parser.h
|
||||||
<string>
|
<string>
|
||||||
<vector>
|
<vector>
|
||||||
"Token.h"
|
"Token.h"
|
||||||
|
|
||||||
|
1351512311 source:/home/attero/development/sencha-lang/sencha/AppleTree.cpp
|
||||||
|
"AppleTree.h"
|
||||||
|
|
||||||
|
1351512311 /home/attero/development/sencha-lang/sencha/AppleTree.h
|
||||||
|
|
||||||
|
|
|
@ -1,16 +1,22 @@
|
||||||
<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
|
<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
|
||||||
<CodeBlocks_layout_file>
|
<CodeBlocks_layout_file>
|
||||||
<ActiveTarget name="Debug" />
|
<ActiveTarget name="Debug" />
|
||||||
<File name="Lexer.cpp" open="1" top="0" tabpos="5">
|
<File name="Lexer.cpp" open="1" top="0" tabpos="4">
|
||||||
<Cursor position="1105" topLine="0" />
|
<Cursor position="1850" topLine="57" />
|
||||||
|
</File>
|
||||||
|
<File name="Lexer.h" open="1" top="0" tabpos="5">
|
||||||
|
<Cursor position="378" topLine="8" />
|
||||||
|
</File>
|
||||||
|
<File name="Parser.cpp" open="1" top="0" tabpos="7">
|
||||||
|
<Cursor position="2767" topLine="98" />
|
||||||
</File>
|
</File>
|
||||||
<File name="Token.cpp" open="1" top="0" tabpos="3">
|
<File name="Token.cpp" open="1" top="0" tabpos="3">
|
||||||
<Cursor position="22" topLine="0" />
|
<Cursor position="22" topLine="0" />
|
||||||
</File>
|
</File>
|
||||||
<File name="Token.h" open="1" top="0" tabpos="2">
|
<File name="Token.h" open="1" top="0" tabpos="2">
|
||||||
<Cursor position="327" topLine="0" />
|
<Cursor position="416" topLine="0" />
|
||||||
</File>
|
</File>
|
||||||
<File name="main.cpp" open="1" top="1" tabpos="1">
|
<File name="main.cpp" open="1" top="1" tabpos="1">
|
||||||
<Cursor position="269" topLine="1" />
|
<Cursor position="1265" topLine="29" />
|
||||||
</File>
|
</File>
|
||||||
</CodeBlocks_layout_file>
|
</CodeBlocks_layout_file>
|
||||||
|
|
Loading…
Reference in New Issue