fix: numbers before a dot pushed to buffers.1, (fixes #1)

atahabaki · Aug 7, 2023 · 26737d2 · 26737d2
1 parent 5c0b5c2
commit 26737d2
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
@@ -182,7 +182,13 @@ pub fn tokenize(content: &str) -> Result<Vec<Token>, TokenizationError> {
                             iter = r_iter;
                             continue;
                         }
-                        _ => buffers.0.push(c),
+                        _ => {
+                            if !buffers.1.is_empty() {
+                                tokens.push(Token::Number(buffers.1.clone(), i - buffers.1.len()));
+                                buffers.1.clear();
+                            }
+                            buffers.0.push(c);
+                        }
                     }
                 } else {
                     buffers.0.push(c);