File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -219,16 +219,19 @@ def test_serialization
219219
220220 as_pretty_str = tokenizer . to_s ( pretty : true )
221221 assert_equal 29163 , as_pretty_str . count ( "\n " )
222- pretty_path = "/tmp/pretty-tokenizer.json"
223- tokenizer . save ( pretty_path , pretty : true )
224222
225- # Compare file content
226- pretty_from_file = File . read ( pretty_path )
227- assert_equal as_pretty_str , pretty_from_file
223+ Dir . mktmpdir do | tmpdir |
224+ pretty_path = " #{ tmpdir } /pretty-tokenizer.json"
225+ tokenizer . save ( pretty_path , pretty : true )
228226
229- new_tokenizer = Tokenizers . from_file ( pretty_path )
230- assert_equal preserialization_size_with_added_tokens , new_tokenizer . vocab_size
231- assert_equal 28996 , new_tokenizer . vocab [ "mellifluous" ]
227+ # Compare file content
228+ pretty_from_file = File . read ( pretty_path )
229+ assert_equal as_pretty_str , pretty_from_file
230+
231+ new_tokenizer = Tokenizers . from_file ( pretty_path )
232+ assert_equal preserialization_size_with_added_tokens , new_tokenizer . vocab_size
233+ assert_equal 28996 , new_tokenizer . vocab [ "mellifluous" ]
234+ end
232235 end
233236
234237 def test_num_special_tokens_to_add
You can’t perform that action at this time.
0 commit comments