first commit

lexer.v

@@ -0,0 +1,89 @@
+module main
+
+enum TokenType as u8 {
+	integer
+	real
+	operator
+	keyword
+	identifier
+	eof
+	unknown
+}
+
+struct Token {
+	type TokenType
+	text string
+}
+
+fn str_from_toktype(type TokenType) string {
+	return match type {
+		.integer {'integer'}
+		.real {'real'}
+		.operator {'operator'}
+		.keyword {'keyword'}
+		.identifier {'identifier'}
+		.eof {'EOF'}
+		.unknown {'unknown'}
+	}
+}
+
+fn is_delimiter(c u8) bool {
+	return " +-*/,;%<>()[]{}=\n".contains(c.ascii_str())
+}
+
+fn is_operator(c u8) bool {
+	return "+-*/=".contains(c.ascii_str())
+}
+
+fn is_real(str string) bool {
+
+	left, right := str.split_once(".") or {return false}
+	return !right.contains(".") && left.is_int() && right.is_int()
+}
+
+fn is_keyword(str string) bool {
+	return [
+		"void", "int", "real", "if", "else", "while", "break", "fn", "return"
+	].contains(str)
+}
+
+fn print_tok(tok Token) {
+	println("${tok.text:8} (${str_from_toktype(tok.type)})")
+}
+
+fn lex(input string) {
+	mut left := 0
+	mut right := 0
+
+	for (right < input.len && left <= right) {
+		if !is_delimiter(input[right]) {
+			right++
+		}
+		if right >= input.len {
+			break
+		}
+		if is_delimiter(input[right]) && left == right {
+			if is_operator(input[right]) {
+				print_tok(Token{TokenType.operator, input[right].ascii_str()})
+			}
+			right++
+			left = right
+		}
+		else if (is_delimiter(input[right]) && left != right) || (right == input.len && left != right) {
+			subs := input.substr(left, right)
+			if is_keyword(subs) {
+				print_tok(Token{TokenType.keyword, subs})
+			} else if subs.is_int() {
+				print_tok(Token{TokenType.integer, subs})
+			} else if is_real(subs) {
+				print_tok(Token{TokenType.real, subs})
+			} else if subs.is_identifier() {
+				print_tok(Token{TokenType.identifier, subs})
+			} else if !subs.is_identifier() && !is_delimiter(input[right-1])  {
+				print_tok(Token{TokenType.unknown, subs})
+			}
+			left = right
+		}
+	}
+	print_tok(Token{TokenType.eof, "EOF"})
+}