Fixed caching

Author: ankane

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.6.4 (unreleased)
+
+- Fixed caching
+
 ## 0.6.3 (2026-01-05)
 
 - Updated Tokenizers to 0.22.2

lib/tokenizers/from_pretrained.rb

@@ -55,7 +55,7 @@ def cached_path(cache_dir, url, headers, options)
         resource_path = File.join(cache_dir, "#{fsum}.#{esum}")
         if File.exist?(resource_path)
           res = head_request(url, headers, options)
-          if res["etag"] == etag
+          if normalize_etag(res["etag"]) == normalize_etag(etag)
             return resource_path
           end
         end
@@ -106,6 +106,10 @@ def head_request(url, headers, options, redirects = 0)
       res
     end
 
+    def normalize_etag(etag)
+      etag.delete_prefix("W/").delete_prefix('"').delete_suffix('"') if etag
+    end
+
     def cache_dir
       cache_dir =
         if ENV["TOKENIZERS_CACHE"]