Ensure the parsing works properly with different locales

libvmaf/src/meson.build

@@ -30,6 +30,24 @@ if cc.symbols_have_underscore_prefix()
     cdata_asm.set10('PREFIX', true)
 endif
 
+# Check for locale handling functions
+uselocale_test = '''
+#define _GNU_SOURCE
+#include <locale.h>
+int main() {
+    locale_t loc = uselocale((locale_t)0);
+    return 0;
+}
+'''
+if cc.compiles(uselocale_test, name: 'uselocale availability')
+    cdata.set('HAVE_USELOCALE', 1)
+endif
+
+# Check for xlocale.h header (needed on some older macOS versions)
+if cc.has_header('xlocale.h')
+    cdata.set('HAVE_XLOCALE_H', 1)
+endif
+
 is_asm_enabled = get_option('enable_asm') == true
 is_cuda_enabled = get_option('enable_cuda') == true
 is_avx512_enabled = get_option('enable_avx512') == true
@@ -529,6 +547,7 @@ libvmaf_sources = [
     src_dir + 'log.c',
     src_dir + 'framesync.c',
     src_dir + 'metadata_handler.c',
+    src_dir + 'thread_locale.c',
 ]
 
 if is_cuda_enabled

libvmaf/src/output.c

@@ -22,6 +22,7 @@
 
 #include "feature/alias.h"
 #include "feature/feature_collector.h"
+#include "thread_locale.h"
 
 #include "libvmaf/libvmaf.h"
 
@@ -72,6 +73,8 @@ int vmaf_write_output_xml(VmafContext *vmaf, VmafFeatureCollector *fc,
     if (!fc) return -EINVAL;
     if (!outfile) return -EINVAL;
 
+    VmafThreadLocaleState* locale_state = vmaf_thread_locale_push_c();
+
     fprintf(outfile, "<VMAF version=\"%s\">\n", vmaf_version());
     fprintf(outfile, "  <params qualityWidth=\"%d\" qualityHeight=\"%d\" />\n",
             width, height);
@@ -154,13 +157,17 @@ int vmaf_write_output_xml(VmafContext *vmaf, VmafFeatureCollector *fc,
 
     fprintf(outfile, "</VMAF>\n");
 
+    vmaf_thread_locale_pop(locale_state);
+
     return 0;
 }
 
 int vmaf_write_output_json(VmafContext *vmaf, VmafFeatureCollector *fc,
                            FILE *outfile, unsigned subsample, double fps,
                            unsigned pic_cnt)
 {
+    VmafThreadLocaleState* locale_state = vmaf_thread_locale_push_c();
+
     int leading_zeros_count;
     fprintf(outfile, "{\n");
     fprintf(outfile, "  \"version\": \"%s\",\n", vmaf_version());
@@ -299,12 +306,16 @@ int vmaf_write_output_json(VmafContext *vmaf, VmafFeatureCollector *fc,
     fprintf(outfile, "\n  }\n");
     fprintf(outfile, "}\n");
 
+    vmaf_thread_locale_pop(locale_state);
+
     return 0;
 }
 
 int vmaf_write_output_csv(VmafFeatureCollector *fc, FILE *outfile,
                            unsigned subsample)
 {
+    VmafThreadLocaleState* locale_state = vmaf_thread_locale_push_c();
+
     int leading_zeros_count;
     fprintf(outfile, "Frame,");
     for (unsigned i = 0; i < fc->cnt; i++) {
@@ -342,12 +353,16 @@ int vmaf_write_output_csv(VmafFeatureCollector *fc, FILE *outfile,
         fprintf(outfile, "\n");
     }
 
+    vmaf_thread_locale_pop(locale_state);
+
     return 0;
 }
 
 int vmaf_write_output_sub(VmafFeatureCollector *fc, FILE *outfile,
                           unsigned subsample)
 {
+    VmafThreadLocaleState* locale_state = vmaf_thread_locale_push_c();
+
     int leading_zeros_count;
     for (unsigned i = 0 ; i < max_capacity(fc); i++) {
         if ((subsample > 1) && (i % subsample))
@@ -381,5 +396,7 @@ int vmaf_write_output_sub(VmafFeatureCollector *fc, FILE *outfile,
         fprintf(outfile, "\n");
     }
 
+    vmaf_thread_locale_pop(locale_state);
+
     return 0;
 }

libvmaf/src/read_json_model.c

@@ -20,6 +20,7 @@
 #include "model.h"
 #include "pdjson.h"
 #include "svm.h"
+#include "thread_locale.h"
 
 #include <errno.h>
 #include <stdlib.h>
@@ -432,6 +433,7 @@ static int model_parse(json_stream *s, VmafModel *model,
 static int vmaf_read_json_model(VmafModel **model, VmafModelConfig *cfg,
                                 json_stream *s)
 {
+    int err = -EINVAL;
     VmafModel *const m = *model = malloc(sizeof(*m));
     if (!m) return -ENOMEM;
     memset(m, 0, sizeof(*m));
@@ -449,7 +451,13 @@ static int vmaf_read_json_model(VmafModel **model, VmafModelConfig *cfg,
     if (!m->score_transform.knots.list) return -ENOMEM;
     memset(m->score_transform.knots.list, 0, knots_sz);
 
-    return model_parse(s, m, cfg->flags);
+    VmafThreadLocaleState* locale_state = vmaf_thread_locale_push_c();
+
+    err = model_parse(s, m, cfg->flags);
+
+    vmaf_thread_locale_pop(locale_state);
+
+    return err;
 }
 
 int vmaf_read_json_model_from_buffer(VmafModel **model, VmafModelConfig *cfg,

libvmaf/src/svm.cpp

@@ -12,6 +12,11 @@
 #include <sstream>
 #include <vector>
 #include "svm.h"
+
+extern "C" {
+#include "thread_locale.h"
+}
+
 int libsvm_version = LIBSVM_VERSION;
 typedef float Qfloat;
 typedef signed char schar;
@@ -2653,11 +2658,7 @@ int svm_save_model(const char *model_file_name, const svm_model *model)
 	FILE *fp = fopen(model_file_name,"w");
 	if(fp==NULL) return -1;
 
-	char *old_locale = setlocale(LC_ALL, NULL);
-	if (old_locale) {
-		old_locale = strdup(old_locale);
-	}
-	setlocale(LC_ALL, "C");
+	VmafThreadLocaleState* locale_state = vmaf_thread_locale_push_c();
 
 	const svm_parameter& param = model->param;
 
@@ -2738,8 +2739,7 @@ int svm_save_model(const char *model_file_name, const svm_model *model)
 		fprintf(fp, "\n");
 	}
 
-	setlocale(LC_ALL, old_locale);
-	free(old_locale);
+	vmaf_thread_locale_pop(locale_state);
 
 	if (ferror(fp) != 0 || fclose(fp) != 0) return -1;
 	else return 0;
@@ -2951,7 +2951,9 @@ class SVMModelParserFileSource {
 	std::ifstream buffer;
 
 public:
-	SVMModelParserFileSource(const char* file_path) : buffer(file_path) {}
+	SVMModelParserFileSource(const char* file_path) : buffer(file_path) {
+		buffer.imbue(std::locale::classic());  // Force C locale for numeric parsing
+	}
 
 	bool read_next(std::string& line) {
 		line.clear();
@@ -2984,7 +2986,10 @@ class SVMModelParserBufferSource {
 	std::istringstream buffer;
 
 public:
-	SVMModelParserBufferSource(const char* buffer, size_t len) : buffer(std::string(buffer, len)) {}
+	SVMModelParserBufferSource(const char* buf, size_t len) 
+		: buffer(std::string(buf, len)) {
+		buffer.imbue(std::locale::classic());  // Force C locale for numeric parsing
+	}
 
 	bool read_next(std::string& line) {
 		line.clear();

libvmaf/src/thread_locale.c

@@ -0,0 +1,113 @@
+/**
+ *
+ *  Copyright 2016-2020 Netflix, Inc.
+ *
+ *     Licensed under the BSD+Patent License (the "License");
+ *     you may not use this file except in compliance with the License.
+ *     You may obtain a copy of the License at
+ *
+ *         https://opensource.org/licenses/BSDplusPatent
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS,
+ *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *     See the License for the specific language governing permissions and
+ *     limitations under the License.
+ *
+ */
+
+#include "thread_locale.h"
+#include "config.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef HAVE_XLOCALE_H
+#include <xlocale.h>
+#endif
+
+#include <locale.h>
+
+// Platform-specific locale state
+struct VmafThreadLocaleState {
+#if defined(HAVE_USELOCALE)
+    // POSIX.1-2008: thread-local locale
+    locale_t c_locale;
+    locale_t old_locale;
+#elif defined(_WIN32)
+    // Windows: thread-local locale via _configthreadlocale
+    int old_per_thread_mode;
+    char old_locale[256];
+#else
+    // No thread-safe locale support available
+    void *reserved; // Reserved for future use
+#endif
+};
+
+VmafThreadLocaleState* vmaf_thread_locale_push_c(void)
+{
+    VmafThreadLocaleState* state = malloc(sizeof(VmafThreadLocaleState));
+    if (!state) return NULL;
+
+    memset(state, 0, sizeof(VmafThreadLocaleState));
+
+#if defined(HAVE_USELOCALE)
+    // POSIX.1-2008: thread-local locale (Linux, macOS, BSD)
+    // Use LC_ALL_MASK for complete locale isolation
+    state->c_locale = newlocale(LC_ALL_MASK, "C", NULL);
+    if (state->c_locale == (locale_t)0) {
+        free(state);
+        return NULL;
+    }
+    state->old_locale = uselocale(state->c_locale);
+
+#elif defined(_WIN32)
+    // Windows: enable per-thread locale, then set to "C"
+    // Use LC_ALL for complete locale isolation
+    state->old_per_thread_mode = _configthreadlocale(_ENABLE_PER_THREAD_LOCALE);
+    if (state->old_per_thread_mode == -1) {
+        free(state);
+        return NULL;
+    }
+
+    const char* old = setlocale(LC_ALL, NULL);
+    if (old) {
+        strncpy(state->old_locale, old, sizeof(state->old_locale) - 1);
+        state->old_locale[sizeof(state->old_locale) - 1] = '\0';
+    }
+
+    setlocale(LC_ALL, "C");
+
+#else
+    // No thread-safe locale support available on this platform
+    free(state);
+    return NULL;
+#endif
+
+    return state;
+}
+
+void vmaf_thread_locale_pop(VmafThreadLocaleState* state)
+{
+    if (!state) return;
+
+#if defined(HAVE_USELOCALE)
+    // POSIX.1-2008: restore thread-local locale
+    if (state->c_locale != (locale_t)0) {
+        uselocale(state->old_locale);
+        freelocale(state->c_locale);
+    }
+
+#elif defined(_WIN32)
+    // Windows: restore locale and per-thread mode
+    if (state->old_locale[0] != '\0') {
+        setlocale(LC_ALL, state->old_locale);
+    }
+
+    if (state->old_per_thread_mode != -1) {
+        _configthreadlocale(state->old_per_thread_mode);
+    }
+#endif
+
+    free(state);
+}

libvmaf/src/thread_locale.h

@@ -0,0 +1,60 @@
+/**
+ *
+ *  Copyright 2016-2020 Netflix, Inc.
+ *
+ *     Licensed under the BSD+Patent License (the "License");
+ *     you may not use this file except in compliance with the License.
+ *     You may obtain a copy of the License at
+ *
+ *         https://opensource.org/licenses/BSDplusPatent
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS,
+ *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *     See the License for the specific language governing permissions and
+ *     limitations under the License.
+ *
+ */
+
+#ifndef __VMAF_SRC_THREAD_LOCALE_H__
+#define __VMAF_SRC_THREAD_LOCALE_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Opaque handle for thread-local locale management.
+ * Stores platform-specific locale state.
+ */
+typedef struct VmafThreadLocaleState VmafThreadLocaleState;
+
+/**
+ * Push "C" locale (all categories) in the current thread.
+ * This ensures consistent I/O behavior regardless of system locale settings:
+ * - Numeric formatting (period as decimal separator)
+ * - Character classification (isalpha, isdigit, etc.)
+ * - String collation and comparison
+ * - Date/time formatting
+ * 
+ * Thread-safe: affects only the calling thread.
+ * 
+ * @return Opaque state handle to be passed to vmaf_thread_locale_pop(),
+ *         or NULL on allocation failure or if thread-safe locale is unavailable.
+ */
+VmafThreadLocaleState* vmaf_thread_locale_push_c(void);
+
+/**
+ * Restore previous locale state in the current thread.
+ * Must be called with the state returned by vmaf_thread_locale_push_c().
+ * 
+ * @param state State handle from vmaf_thread_locale_push_c().
+ *              Freed internally, do not use after this call.
+ */
+void vmaf_thread_locale_pop(VmafThreadLocaleState* state);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __VMAF_SRC_THREAD_LOCALE_H__ */