Updated the scanner and the tokens

sample.avl

@@ -1,5 +1,5 @@
 # Single-line comments are written using a semicolon
-
+# TIP: read this file with python formatting
 
 # Member Variables
 int x = 8
@@ -79,8 +79,7 @@ func doing_math():
 	math::pow(first, 2) # 64
 	math::sqrt(second) # 2
 
-	# TODO, should they be part of the math namespace?
-	print(PI, TAU, INF, NAN) # built-in constants
+	print(math::PI, math::TAU, math::INF, math::NAN) # built-in constants
 
 
 # Control Flow
@@ -229,7 +228,7 @@ recipe FunFuncExample:
 	# float = 0.0
 	# bool = false
 	# str = ""
-	# *array = []
+	# array = []
 
 	func default_args(str first_name = "stranger", str last_name):
 		print("Nice to meet you, %s %s\n", first_name, last_name)
@@ -281,14 +280,6 @@ recipe Person:
 		print("Goodbye :(")
 		print("\n")
 
-
-func returns_nothing() -> void:
-	# imagine a dragon
-	return
-
-func join(str arg1, str arg2) -> str:
-	return arg1 + arg2
-
 func new_person() -> ref Person:
 	ref Person p("Aaron Franke", 23)
 	return p

src/scanner/scanner.c

@@ -103,20 +103,40 @@ static struct token get_token()
 	case ')': advance(); return GET_TOKEN(TOKEN_RIGHT_PAREN);
 	case '{': advance(); return GET_TOKEN(TOKEN_LEFT_BRACE);
 	case '}': advance(); return GET_TOKEN(TOKEN_RIGHT_PAREN);
+	case '[': advance(); return GET_TOKEN(TOKEN_LEFT_SQUARE);
+	case ']': advance(); return GET_TOKEN(TOKEN_RIGHT_SQUARE);
 	case ',': advance(); return GET_TOKEN(TOKEN_COMMA);
 	case '.': advance(); return GET_TOKEN(TOKEN_DOT);
-	case '-': advance(); return GET_TOKEN(TOKEN_MINUS);
 	case '+': advance(); return GET_TOKEN(TOKEN_PLUS);
-	case ';': advance(); return GET_TOKEN(TOKEN_SEMICOLON);
 	case '*': advance(); return GET_TOKEN(TOKEN_STAR);
 	case '/': advance(); return GET_TOKEN(TOKEN_SLASH);
 	case '\0': return GET_TOKEN(TOKEN_END_OF_FILE);
+	case '\n':
+		line++;
+		return GET_TOKEN(TOKEN_NEWLINE);
+	case '\t':
+		return GET_TOKEN(TOKEN_TAB);
+		/* Comment */
 	case '#':
 		advance();
 		while (current[0] != '\n' && current[0] != '\0') advance();
 		return get_token();
 
 		/* one or two character tokens */
+	case '-':
+		advance();
+		if (current[0] == '>') {
+			advance();
+			return GET_TOKEN(TOKEN_ARROW); 
+		}
+		return GET_TOKEN(TOKEN_MINUS);
+	case ':':
+		advance();
+		if (current[0] == ':') {
+			advance();
+			return GET_TOKEN(TOKEN_COLON_COLON);
+		}
+		return GET_TOKEN(TOKEN_COLON);
 	case '!':
 		advance();
 		if (current[0] == '=') {
@@ -147,10 +167,7 @@ static struct token get_token()
 		return GET_TOKEN(TOKEN_LESS);
 
 		/* ignored */
-	case '\n':
-		line++;
 	case ' ':
-	case '\t':
 	case '\f':
 	case '\v':
 	case '\r':
@@ -244,21 +261,21 @@ enum token_type get_keyword_type(const char *str)
 {
 #define GET(kwstr, tk) if (strcmp(str, kwstr) == 0) return tk;
 
-	GET("blueprint", TOKEN_BLUEPRINT);
-	GET("con", TOKEN_CON);
-	GET("dis", TOKEN_DIS);
-	GET("else", TOKEN_ELSE);
-	GET("fluid", TOKEN_FLUID);
-	GET("if", TOKEN_IF);
-	GET("me", TOKEN_ME);
-	GET("nil", TOKEN_NIL);
-	GET("no", TOKEN_NO);
-	GET("procedure", TOKEN_PROCEDURE);
-	GET("produce", TOKEN_PRODUCE);
-	GET("solid", TOKEN_SOLID);
-	GET("while", TOKEN_WHILE);
-	GET("write", TOKEN_WRITE);
-	GET("yes", TOKEN_YES);
+	/* GET("blueprint", TOKEN_BLUEPRINT); */
+	/* GET("con", TOKEN_CON); */
+	/* GET("dis", TOKEN_DIS); */
+	/* GET("else", TOKEN_ELSE); */
+	/* GET("fluid", TOKEN_FLUID); */
+	/* GET("if", TOKEN_IF); */
+	/* GET("me", TOKEN_ME); */
+	/* GET("nil", TOKEN_NIL); */
+	/* GET("no", TOKEN_NO); */
+	/* GET("procedure", TOKEN_PROCEDURE); */
+	/* GET("produce", TOKEN_PRODUCE); */
+	/* GET("solid", TOKEN_SOLID); */
+	/* GET("while", TOKEN_WHILE); */
+	/* GET("write", TOKEN_WRITE); */
+	/* GET("yes", TOKEN_YES); */
 
 	return TOKEN_IDENTIFIER;
 #undef GET

src/scanner/token.h

@@ -30,8 +30,11 @@ enum token_type{
 	TOKEN_LEFT_PAREN, TOKEN_RIGHT_PAREN, TOKEN_LEFT_BRACE,
 	TOKEN_RIGHT_BRACE, TOKEN_COMMA, TOKEN_DOT, TOKEN_MINUS,
 	TOKEN_PLUS, TOKEN_SEMICOLON, TOKEN_STAR, TOKEN_SLASH,
+	TOKEN_LEFT_SQUARE, TOKEN_RIGHT_SQUARE, TOKEN_NEWLINE,
+	TOKEN_TAB, TOKEN_ARROW,
 
 	/* one or two character tokens */
+	TOKEN_COLON, TOKEN_COLON_COLON,
 	TOKEN_BANG, TOKEN_BANG_EQUAL,
 	TOKEN_EQUAL, TOKEN_EQUAL_EQUAL,
 	TOKEN_GREATER, TOKEN_GREATER_EQUAL,
@@ -41,9 +44,12 @@ enum token_type{
 	TOKEN_IDENTIFIER, TOKEN_STRING, TOKEN_NUMBER,
 
 	/* keywords */
-	TOKEN_CON, TOKEN_DIS, TOKEN_BLUEPRINT, TOKEN_IF, TOKEN_ELSE,
-	TOKEN_YES, TOKEN_NO, TOKEN_PROCEDURE, TOKEN_WRITE, TOKEN_PRODUCE,
-	TOKEN_ME, TOKEN_WHILE, TOKEN_FLUID, TOKEN_SOLID, TOKEN_NIL,
+	TOKEN_IF, TOKEN_ELSE, TOKEN_WHILE, TOKEN_INT, TOKEN_BYTE,
+	TOKEN_UINT, TOKEN_SBYTE, TOKEN_FLOAT, TOKEN_BOOL, TOKEN_TRUE,
+	TOKEN_FALSE, TOKEN_STR, TOKEN_ARRAY, TOKEN_MAP, TOKEN_CONST,
+	TOKEN_ENUM, TOKEN_FUNC, TOKEN_RETURN, TOKEN_CONTINUE,
+	TOKEN_BREAK, TOKEN_PASS, TOKEN_PRINT, TOKEN_PRINT_ERR,
+	TOKEN_AS, TOKEN_REF,TOKEN_RECIPE,
 
 	TOKEN_INVALID, TOKEN_END_OF_FILE
 };