Now, lexer can deal with character literals with white spaces, hurray!
parent
78c4572f5e
commit
67b2379d03
|
@ -2,8 +2,8 @@
|
|||
|
||||
Lexer::Lexer()
|
||||
{
|
||||
string keys[] = {"function", "class", "penis"};
|
||||
keywords.assign(keys, keys+3);
|
||||
string keys[] = {"function", "class", "for", "while", "if", "else"};
|
||||
keywords.assign(keys, keys+6);
|
||||
|
||||
char punct[] = {'.', ',', ';', '{', '}', '[', ']', '(', ')'};
|
||||
punctuation.assign(punct, punct+9);
|
||||
|
@ -62,6 +62,7 @@ pair<string, Token> Lexer::parse_token(string line)
|
|||
{
|
||||
string token_value = "";
|
||||
unsigned int i;
|
||||
bool in_char_literal = false;
|
||||
for(i=0; i< line.size(); i++)
|
||||
{
|
||||
if(token_value == "" && isspace(line[i])) continue;
|
||||
|
@ -69,6 +70,14 @@ pair<string, Token> Lexer::parse_token(string line)
|
|||
if(isalnum(line[i]) || line[i] == '\"' || line[i]== '_')
|
||||
{
|
||||
token_value += line[i];
|
||||
if(line[i] == '\"')
|
||||
{
|
||||
if(in_char_literal)
|
||||
{
|
||||
in_char_literal = false;
|
||||
}
|
||||
else in_char_literal = true;
|
||||
}
|
||||
}
|
||||
else if(ispunct(line[i]))
|
||||
{
|
||||
|
@ -88,6 +97,10 @@ pair<string, Token> Lexer::parse_token(string line)
|
|||
}
|
||||
break;
|
||||
}
|
||||
else if (in_char_literal && isspace(line[i]))
|
||||
{
|
||||
token_value += line[i];
|
||||
}
|
||||
else break;
|
||||
}
|
||||
|
||||
|
|
|
@ -14,13 +14,7 @@ class Lexer
|
|||
Lexer();
|
||||
virtual ~Lexer();
|
||||
|
||||
bool is_keyword(string value);
|
||||
bool is_punctuation(char c);
|
||||
bool is_operator(string value );
|
||||
|
||||
vector<string> keywords;
|
||||
vector<char> punctuation;
|
||||
vector<string> operators;
|
||||
|
||||
void add_keyword(string word);
|
||||
void add_punctuation_char(char c);
|
||||
|
@ -32,6 +26,14 @@ class Lexer
|
|||
|
||||
protected:
|
||||
private:
|
||||
|
||||
bool is_keyword(string value);
|
||||
bool is_punctuation(char c);
|
||||
bool is_operator(string value );
|
||||
|
||||
vector<string> keywords;
|
||||
vector<char> punctuation;
|
||||
vector<string> operators;
|
||||
};
|
||||
|
||||
#endif // LEXER_H
|
||||
|
|
|
@ -3,7 +3,6 @@
|
|||
|
||||
Parser::Parser(vector<Token> tokens)
|
||||
{
|
||||
//token_stream.push_back(Token(t_symbol, "dupa"));
|
||||
error_message = "***ERRORS DURING PARSING***\n";
|
||||
report_message = "***PARSER REPORT***\n";
|
||||
token_stream = tokens;
|
||||
|
@ -140,7 +139,6 @@ void Parser::statement()
|
|||
{
|
||||
if(accept("{"))
|
||||
{
|
||||
report("GO Deeper\n");
|
||||
while(!accept("}"))
|
||||
{
|
||||
statement();
|
||||
|
|
|
@ -10,23 +10,26 @@ class Parser
|
|||
public:
|
||||
Parser(vector<Token> tokens);
|
||||
virtual ~Parser();
|
||||
void interpret();
|
||||
string report_message;
|
||||
string error_message;
|
||||
|
||||
protected:
|
||||
private:
|
||||
Token current_token;
|
||||
string tok_value;
|
||||
vector<Token> token_stream;
|
||||
int position_in_stream;
|
||||
|
||||
string report_message;
|
||||
string error_message;
|
||||
|
||||
bool read_next();
|
||||
bool peek(string s);
|
||||
bool accept(string s);
|
||||
bool expect(string s);
|
||||
bool is_type();
|
||||
|
||||
void error(string s);
|
||||
void report(string s);
|
||||
void interpret();
|
||||
|
||||
void statement();
|
||||
|
||||
void add_expr();
|
||||
|
@ -35,10 +38,6 @@ class Parser
|
|||
void rel_expr();
|
||||
void eq_expr();
|
||||
void expr();
|
||||
|
||||
bool is_type();
|
||||
protected:
|
||||
private:
|
||||
};
|
||||
|
||||
#endif // PARSER_H
|
||||
|
|
Binary file not shown.
|
@ -8,7 +8,7 @@ using namespace std;
|
|||
|
||||
void test_lexer()
|
||||
{
|
||||
string test_line = "def i; bulb; i + 3; string banan; banan = \"banan\"; string kaboom(num how_many_times) { def z; }";
|
||||
string test_line = "def i; bulb; i + 3; string banan; banan = \"banan and other stuff\"; string kaboom(num how_many_times) { def z; }";
|
||||
string test_line2 = "def how_many_trees = 1; how_many_trees + 3 == 2; num cut_tree( num how_many) {return how_many -1; != <=}";
|
||||
Lexer lexer;
|
||||
vector<Token> tokens = lexer.parse_line(test_line);
|
||||
|
@ -22,10 +22,10 @@ void test_lexer()
|
|||
|
||||
tokens = lexer.parse_line(test_line2);
|
||||
|
||||
for(int i=0; i< tokens.size(); i++)
|
||||
/*for(int i=0; i< tokens.size(); i++)
|
||||
{
|
||||
cout << tokens[i].get_value() << " type: " << tokens[i].get_type() << endl;
|
||||
}
|
||||
}*/
|
||||
}
|
||||
|
||||
void test_parser()
|
||||
|
@ -52,7 +52,7 @@ void test_parser()
|
|||
int main()
|
||||
{
|
||||
cout << "Hello world!" << endl;
|
||||
test_parser();
|
||||
//test_lexer();
|
||||
//test_parser();
|
||||
test_lexer();
|
||||
return 0;
|
||||
}
|
||||
|
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -34,6 +34,8 @@
|
|||
</Compiler>
|
||||
<Unit filename="Lexer.cpp" />
|
||||
<Unit filename="Lexer.h" />
|
||||
<Unit filename="Parser.cpp" />
|
||||
<Unit filename="Parser.h" />
|
||||
<Unit filename="Token.cpp" />
|
||||
<Unit filename="Token.h" />
|
||||
<Unit filename="main.cpp" />
|
||||
|
|
|
@ -5,10 +5,10 @@
|
|||
1351424620 /home/attero/development/sencha-lang/sencha/Token.h
|
||||
<string>
|
||||
|
||||
1351428918 source:/home/attero/development/sencha-lang/sencha/Lexer.cpp
|
||||
1351540082 source:/home/attero/development/sencha-lang/sencha/Lexer.cpp
|
||||
"Lexer.h"
|
||||
|
||||
1351413156 /home/attero/development/sencha-lang/sencha/Lexer.h
|
||||
1351539503 /home/attero/development/sencha-lang/sencha/Lexer.h
|
||||
<vector>
|
||||
<string>
|
||||
<utility>
|
||||
|
@ -16,19 +16,24 @@
|
|||
<iostream>
|
||||
"Token.h"
|
||||
|
||||
1351429258 source:/home/attero/development/sencha-lang/sencha/main.cpp
|
||||
1351539590 source:/home/attero/development/sencha-lang/sencha/main.cpp
|
||||
<iostream>
|
||||
<string>
|
||||
"Token.h"
|
||||
"Lexer.h"
|
||||
"Parser.h"
|
||||
|
||||
1351429380 source:/home/attero/development/sencha-lang/sencha/Parser.cpp
|
||||
1351539503 source:/home/attero/development/sencha-lang/sencha/Parser.cpp
|
||||
"Parser.h"
|
||||
"iostream"
|
||||
|
||||
1351424740 /home/attero/development/sencha-lang/sencha/Parser.h
|
||||
1351539525 /home/attero/development/sencha-lang/sencha/Parser.h
|
||||
<string>
|
||||
<vector>
|
||||
"Token.h"
|
||||
|
||||
1351512311 source:/home/attero/development/sencha-lang/sencha/AppleTree.cpp
|
||||
"AppleTree.h"
|
||||
|
||||
1351512311 /home/attero/development/sencha-lang/sencha/AppleTree.h
|
||||
|
||||
|
|
|
@ -1,16 +1,22 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
|
||||
<CodeBlocks_layout_file>
|
||||
<ActiveTarget name="Debug" />
|
||||
<File name="Lexer.cpp" open="1" top="0" tabpos="5">
|
||||
<Cursor position="1105" topLine="0" />
|
||||
<File name="Lexer.cpp" open="1" top="0" tabpos="4">
|
||||
<Cursor position="1850" topLine="57" />
|
||||
</File>
|
||||
<File name="Lexer.h" open="1" top="0" tabpos="5">
|
||||
<Cursor position="378" topLine="8" />
|
||||
</File>
|
||||
<File name="Parser.cpp" open="1" top="0" tabpos="7">
|
||||
<Cursor position="2767" topLine="98" />
|
||||
</File>
|
||||
<File name="Token.cpp" open="1" top="0" tabpos="3">
|
||||
<Cursor position="22" topLine="0" />
|
||||
</File>
|
||||
<File name="Token.h" open="1" top="0" tabpos="2">
|
||||
<Cursor position="327" topLine="0" />
|
||||
<Cursor position="416" topLine="0" />
|
||||
</File>
|
||||
<File name="main.cpp" open="1" top="1" tabpos="1">
|
||||
<Cursor position="269" topLine="1" />
|
||||
<Cursor position="1265" topLine="29" />
|
||||
</File>
|
||||
</CodeBlocks_layout_file>
|
||||
|
|
Loading…
Reference in New Issue