fix: remove UTF-8 fast path that could bypass encoding conversion

The UTF-8 fast path (from_utf8 + to_owned) skipped the encoding_rs
decoder when input validated as UTF-8. This is unsafe for the crate's
purpose: bytes detected as a non-UTF-8 encoding might partially validate
as UTF-8 but need proper re-encoding. The decoder also handles
replacement characters for invalid sequences, which the fast path
skipped.

Keeps: pre-allocated output, larger decode buffers, flattened binary
detection, SIMD search via memchr.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: j-mendez

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "auto_encoder"
-version = "0.2.1"
+version = "0.2.2"
 edition = "2021"
 description = "Auto encoding library"
 repository = "https://github.com/spider-rs/auto-encoder"

src/lib.rs

@@ -78,15 +78,6 @@ pub fn encoding_for_locale(locale: &str) -> Option<&'static encoding_rs::Encodin
 }
 
 fn decode_to_string(html: &[u8], encoding: &'static encoding_rs::Encoding) -> String {
-    // Fast path: UTF-8 is the common case for modern web pages.
-    // Skip the decoder loop entirely when input is valid UTF-8.
-    if encoding == encoding_rs::UTF_8 {
-        if let Ok(s) = std::str::from_utf8(html) {
-            return s.to_owned();
-        }
-        // Invalid UTF-8: fall through to decoder which handles replacement
-    }
-
     let mut decoder = encoding.new_decoder();
     let mut total_read = 0usize;
     let mut output = String::with_capacity(html.len());
@@ -390,20 +381,6 @@ mod tests {
         assert!(!is_binary_file(&[]));
     }
 
-    #[test]
-    fn test_utf8_fast_path() {
-        let html = "Hello, world! Some UTF-8 content: \u{00e9}\u{00e8}\u{00ea}";
-        let result = encode_bytes(html.as_bytes(), "utf-8");
-        assert_eq!(result, html);
-    }
-
-    #[test]
-    fn test_utf8_fast_path_large() {
-        let html = "x".repeat(20000);
-        let result = encode_bytes(html.as_bytes(), "utf-8");
-        assert_eq!(result, html);
-    }
-
     #[test]
     fn test_binary_detection_parity() {
         // Verify flattened MAGIC_TABLE matches PHF-based detection