v0.22.0

benbrandt · Jan 17, 2025 · 217fb50 · 217fb50
1 parent 4a72472
commit 217fb50
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,12 +1,18 @@
 # Changelog
 
-## v0.20.2
+## v0.22.0
 
-### What's New
+### Breaking Changes
 
-#### Python
+- Revert change to special token behavior in v0.21. This had many unintended side effects, and does not seem to be recommended for chunking.
+
+## v0.21.0
 
-- Minor release to include latest pyo3 and tree-sitter dependencies.
+### Breaking Changes
+
+- Special tokens are now also encoded by both Huggingface and Tiktoken tokenizers. This is closer to the default behavior on the Python side, and should make sure if a model adds tokens at the beginning or end of a sequence, these are accounted for as well. This is especially important for embedding models that can add a special token to the beginning of the sequence, and the chunks generated didn't actually fit within the context window because of this.
+
+### What's New
 
 #### Rust
 

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -2,7 +2,7 @@
 members = ["bindings/*"]
 
 [workspace.package]
-version = "0.20.2"
+version = "0.22.0"
 authors = ["Ben Brandt <[email protected]>"]
 edition = "2021"
 description = "Split text into semantic chunks, up to a desired chunk size. Supports calculating length by characters and tokens, and is callable from Rust and Python."