feat(decode): Add opt-in lossy mode and decode_bytes

README.md

@@ -17,17 +17,15 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
-Usage should be very similar to the python library. Here's a simple example
-
-Encode and decode text
+Encode and decode text:
 
 ```ruby
 require 'tiktoken_ruby'
 enc = Tiktoken.get_encoding("cl100k_base")
 enc.decode(enc.encode("hello world")) #=> "hello world"
 ```
 
-Encoders can also be retrieved by model name
+Encoders can also be retrieved by model name:
 
 ```ruby
 require 'tiktoken_ruby'
@@ -69,6 +67,27 @@ enc.encode_with_special_tokens(text)
 
 All methods round-trip correctly through `decode`.
 
+### Decoding methods
+
+- `decode(tokens, errors: :strict)` - Decodes tokens back into a UTF-8 string
+- `decode_bytes(tokens)` - Decodes tokens into their raw bytes (an `ASCII-8BIT` string), without UTF-8 validation
+
+Because BPE tokens are byte-level, a single character (an emoji, or non-Latin scripts) can span multiple tokens. Truncating a token array (like, "trim text to N tokens") can leave a prefix that is not valid UTF-8. The `errors:` option controls how those invalid sequences are handled.
+
+```ruby
+enc = Tiktoken.encoding_for_model("gpt-4o")
+tokens = enc.encode("🦄") # the emoji spans multiple tokens
+
+# :strict (default) - raise Tiktoken::UnicodeError on invalid UTF-8
+enc.decode(tokens.first(2))                    #=> raises Tiktoken::UnicodeError
+
+# :replace - substitute invalid sequences with "�" (matches Python tiktoken's default)
+enc.decode(tokens.first(2), errors: :replace)  #=> "�"
+
+# decode_bytes - get the raw bytes and handle them yourself
+enc.decode_bytes(tokens.first(2))              #=> "\xF0\x9F\xA6" (ASCII-8BIT)
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

ext/tiktoken_ruby/src/core_bpe_wrapper.rs

@@ -3,7 +3,7 @@ use std::ffi::c_void;
 
 use tiktoken_rs::Rank;
 
-use crate::uncicode_error;
+use crate::unicode_error;
 
 #[magnus::wrap(class = "Tiktoken::Ext::CoreBPE")]
 pub struct CoreBPEWrapper {
@@ -35,6 +35,12 @@ struct DecodeData {
     result: Result<String, String>,
 }
 
+struct DecodeBytesData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    ids: Vec<Rank>,
+    result: Result<Vec<u8>, String>,
+}
+
 unsafe extern "C" fn encode_ordinary_without_gvl(data: *mut c_void) -> *mut c_void {
     let data = &mut *(data as *mut EncodeOrdinaryData);
     let core_bpe = &*data.core_bpe;
@@ -64,6 +70,13 @@ unsafe extern "C" fn decode_without_gvl(data: *mut c_void) -> *mut c_void {
     std::ptr::null_mut()
 }
 
+unsafe extern "C" fn decode_bytes_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut DecodeBytesData);
+    let core_bpe = &*data.core_bpe;
+    data.result = core_bpe.decode_bytes(&data.ids).map_err(|e| e.to_string());
+    std::ptr::null_mut()
+}
+
 impl CoreBPEWrapper {
     pub fn new(core_bpe: tiktoken_rs::CoreBPE) -> Self {
         Self { core_bpe }
@@ -150,12 +163,45 @@ impl CoreBPEWrapper {
         }
 
         data.result.map_err(|e| {
-            let error = match uncicode_error() {
+            let error = match unicode_error() {
                 Ok(error) => error,
                 Err(e) => return e,
             };
 
             magnus::Error::new(error, e)
         })
     }
+
+    pub fn decode_bytes(&self, ids: Vec<Rank>) -> Result<magnus::RString, magnus::Error> {
+        let mut data = DecodeBytesData {
+            core_bpe: &self.core_bpe as *const _,
+            ids,
+            result: Err(String::new()),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(decode_bytes_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        // Build the Ruby string only after the GVL has been re-acquired.
+        // `str_from_slice` yields an ASCII-8BIT (binary) string, so the bytes
+        // are handed back verbatim without any UTF-8 validation. We hold the
+        // GVL here (this is called from Ruby), so `Ruby::get` cannot fail.
+        let ruby = magnus::Ruby::get().unwrap();
+        data.result
+            .map(|bytes| ruby.str_from_slice(&bytes))
+            .map_err(|e| {
+                let error = match unicode_error() {
+                    Ok(error) => error,
+                    Err(e) => return e,
+                };
+
+                magnus::Error::new(error, e)
+            })
+    }
 }

ext/tiktoken_ruby/src/lib.rs

@@ -34,7 +34,7 @@ fn module() -> Result<RModule, magnus::Error> {
     Ruby::get().unwrap().define_module("Tiktoken")
 }
 
-fn uncicode_error() -> Result<ExceptionClass, magnus::Error> {
+fn unicode_error() -> Result<ExceptionClass, magnus::Error> {
     module()?.define_error(
         "UnicodeError",
         Ruby::get().unwrap().exception_standard_error(),
@@ -45,6 +45,10 @@ fn uncicode_error() -> Result<ExceptionClass, magnus::Error> {
 fn init() -> Result<(), Error> {
     let module = module()?;
 
+    // Define Tiktoken::UnicodeError eagerly so the constant exists as soon as
+    // the extension is loaded, rather than only after the first decode error.
+    unicode_error()?;
+
     let factory_module = module.define_module("BpeFactory")?;
     factory_module.define_singleton_method("r50k_base", function!(r50k_base, 0))?;
     factory_module.define_singleton_method("p50k_base", function!(p50k_base, 0))?;
@@ -67,5 +71,6 @@ fn init() -> Result<(), Error> {
     )?;
 
     bpe_class.define_method("decode", method!(CoreBPEWrapper::decode, 1))?;
+    bpe_class.define_method("decode_bytes", method!(CoreBPEWrapper::decode_bytes, 1))?;
     Ok(())
 }

lib/tiktoken_ruby/encoding.rb

@@ -47,11 +47,41 @@ def encode_with_special_tokens(text)
     @ext_base_bpe.encode_with_special_tokens(text)
   end
 
-  # Decodes the tokens back into text
+  # Decodes the tokens back into text.
+  #
+  # BPE tokens are byte-level, so a single character (an emoji, non-Latin
+  # scripts) can span multiple tokens. Truncating a token array to a limit can
+  # therefore leave a prefix that is not valid UTF-8. The +errors+ option
+  # controls how those invalid byte sequences are handled:
+  #
+  # * +:strict+ (default) - raise Tiktoken::UnicodeError on invalid UTF-8.
+  # * +:replace+ - substitute each invalid sequence with the Unicode replacement
+  #   character (U+FFFD, +�+), matching Python tiktoken's default behavior.
+  #
   # @param tokens [Array<Integer>] The tokens to decode
-  # @return [String] The decoded text
-  def decode(tokens)
-    @ext_base_bpe.decode(tokens)
+  # @param errors [Symbol] How to handle invalid UTF-8 (:strict or :replace)
+  # @return [String] The decoded text (UTF-8)
+  def decode(tokens, errors: :strict)
+    case errors
+    when :strict
+      @ext_base_bpe.decode(tokens)
+    when :replace
+      decode_bytes(tokens).force_encoding(Encoding::UTF_8).scrub("\u{FFFD}")
+    else
+      raise ArgumentError, "errors must be :strict or :replace, got #{errors.inspect}"
+    end
+  end
+
+  # Decodes the tokens back into their raw bytes, without any UTF-8 validation.
+  #
+  # The returned string has ASCII-8BIT (binary) encoding and is not guaranteed
+  # to be valid UTF-8 — useful when you want to handle invalid sequences
+  # yourself. Matches Python tiktoken's +decode_bytes+.
+  #
+  # @param tokens [Array<Integer>] The tokens to decode
+  # @return [String] The decoded bytes (ASCII-8BIT)
+  def decode_bytes(tokens)
+    @ext_base_bpe.decode_bytes(tokens)
   end
 
   private

spec/tiktoken_ruby_spec.rb

@@ -77,6 +77,65 @@
     end
   end
 
+  describe "decoding truncated multi-byte characters" do
+    let(:encoding) { Tiktoken.encoding_for_model("gpt-4o") }
+
+    # An emoji is 4 bytes spread across several tokens, so truncating the token
+    # array leaves a prefix that is not valid UTF-8.
+    let(:tokens) { encoding.encode("🦄") }
+    let(:truncated) { tokens.first(tokens.length - 1) }
+
+    it "encodes the emoji across multiple tokens" do
+      expect(tokens.length).to be > 1
+    end
+
+    it "raises Tiktoken::UnicodeError on invalid UTF-8 by default" do
+      expect { encoding.decode(truncated) }
+        .to raise_error(Tiktoken::UnicodeError)
+    end
+
+    it "raises Tiktoken::UnicodeError with an explicit errors: :strict" do
+      expect { encoding.decode(truncated, errors: :strict) }
+        .to raise_error(Tiktoken::UnicodeError)
+    end
+
+    it "replaces invalid UTF-8 with the replacement character with errors: :replace" do
+      result = encoding.decode(truncated, errors: :replace)
+      expect(result.encoding).to eq(Encoding::UTF_8)
+      expect(result).to include("\u{FFFD}")
+    end
+
+    it "does not raise with errors: :replace" do
+      expect { encoding.decode(truncated, errors: :replace) }.not_to raise_error
+    end
+
+    it "raises ArgumentError for an unknown errors mode" do
+      expect { encoding.decode(truncated, errors: :bogus) }
+        .to raise_error(ArgumentError)
+    end
+
+    it "still round-trips valid tokens exactly" do
+      expect(encoding.decode(tokens)).to eq("🦄")
+    end
+
+    describe "#decode_bytes" do
+      it "returns the raw bytes as an ASCII-8BIT string" do
+        bytes = encoding.decode_bytes(truncated)
+        expect(bytes.encoding).to eq(Encoding::ASCII_8BIT)
+      end
+
+      it "returns bytes that are a valid prefix of the full encoding" do
+        prefix = encoding.decode_bytes(truncated)
+        full = encoding.decode_bytes(tokens)
+        expect(full.b).to start_with(prefix.b)
+      end
+
+      it "round-trips valid tokens to UTF-8 bytes" do
+        expect(encoding.decode_bytes(tokens).force_encoding(Encoding::UTF_8)).to eq("🦄")
+      end
+    end
+  end
+
   describe "special token handling" do
     let(:encoding) { Tiktoken.get_encoding("cl100k_base") }
     let(:text_with_special) { "Hello<|endoftext|>World" }