tokenizer handles strings differently

1 files changed, 14 insertions, 0 deletions

diff --git a/src/main.go b/src/main.go
index d0c54ca..5cb7075 100644
--- a/src/main.go
+++ b/src/main.go
@@ -41,11 +41,19 @@ func tokenize(path string) []Token {
 	for lineIndex := 0; lineIndex < len(lines); lineIndex++ {
 		line := lines[lineIndex]
 		buf := ""
+		isStr := false
 		for offset := 0; offset < len(line); offset++ {
 			char := string(line[offset])
 			if char == "#" {
 				break
 			}
+			if char == "\"" {
+				isStr = !isStr
+			}
+			if isStr {
+				buf += char
+				continue;
+			}
 			if char != " " && char != "\t" {
 				buf += char
 				if offset != len(line) - 1 {
@@ -59,7 +67,13 @@ func tokenize(path string) []Token {
 			tokens = append(tokens, tokenBuf)
 			buf = ""
 		}
+		if isStr {
+			panic("invalid syntax")
+		}
 	}
+
+	print(tokens)
+	os.Exit(0)
 	return tokens
 }