luzer: fix memory leak in FDP

CHANGELOG.md

@@ -57,3 +57,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - A memory leak in a Lua-based implementation of `TestOneInput()`.
 - An initial buffer size in FuzzedDataProvider.
 - Search of the archiver tool (`ar`) in the OSS Fuzz environment.
+- Memory leak in FuzzedDataProvider (#52).
+- Segfault on parsing a broken dictionary (#65).

docs/usage.md

@@ -228,9 +228,33 @@ and can be disabled by setting the enviroment variable
 `DISABLE_LUAJIT_METRICS`. Learn more about the enviroment variable
 in the section [LuaJIT Metrics](#luajit-metrics).
 
+### Memory leaks
+
+When a target application (or a fuzzer) consumes increasing
+amounts of RAM over time without releasing it, it can be a normal
+memory consumption or memory leak is occurring. The common causes
+of memory leak are: unreleased references, improper handling of
+native resources in Lua. If you are encountering a memory leak in
+a target application while using luzer, you may need to use memory
+debugging tools to identify the specific code segment that isn't
+freeing memory.
+
+luzer can encounter false positive memory leaks during testing.
+When fuzzing native extensions, using FDP (FuzzingDataProvider), or
+using FFI memory leak detection (ASan) should be disabled to
+prevent false reports. Set flag `-detect_leaks=0` using enviroment
+variable [`ASAN_OPTIONS`][asan-flags] as it is recommended by
+AddressSanitizer:
+
+```
+SUMMARY: AddressSanitizer: 96 byte(s) leaked in 6 allocation(s).
+INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.
+```
+
 [ffi-library-url]: https://luajit.org/ext_ffi.html
 [programming-in-lua-8]: https://www.lua.org/pil/8.html
 [programming-in-lua-24]: https://www.lua.org/pil/24.html
 [atheris-native-extensions]: https://github.com/google/atheris/blob/master/native_extension_fuzzing.md
 [atheris-native-extensions-video]: https://www.youtube.com/watch?v=oM-7lt43-GA
 [luacov-website]: https://lunarmodules.github.io/luacov/
+[asan-flags]: https://github.com/google/sanitizers/wiki/addresssanitizerflags

luzer/init.lua

@@ -68,7 +68,20 @@ local function Fuzz(test_one_input, custom_mutator, func_args)
     if type(luzer_args) ~= "table" then
         error("args is not a table")
     end
+
     local flags = build_flags(arg, luzer_args)
+
+    -- Lua has the GC-based memory management model, it may
+    -- accumulate memory between the TestOneInput() runs.
+    -- libFuzzer has a flag `detect_leaks`, it is enabled by
+    -- default, when the option is enabled and if LeakSanitizer is
+    -- enabled it tries to detect memory leaks during fuzzing
+    -- (i.e. not only at shut down). The option `detect_leaks` may
+    -- lead to false positives, so it is disabled by default.
+    if flags["detect_leaks"] == nil then
+        flags["detect_leaks"] = 0
+    end
+
     local test_path = arg[0]
     local lua_bin = progname(arg)
     local test_cmd = ("%s %s"):format(lua_bin, test_path)

luzer/luzer.c

@@ -437,6 +437,33 @@ free_argv(int argc, char **argv)
 	free(argv);
 }
 
+NO_SANITIZE static void
+shutdown_lua(void)
+{
+	lua_State *L = get_global_lua_state();
+	luaL_cleanup(L);
+	lua_close(L);
+	set_global_lua_state(NULL);
+}
+
+int atexit_retcode;
+
+NO_SANITIZE void
+atexit_handler(void) {
+	_exit(atexit_retcode);
+}
+
+NO_SANITIZE static void
+graceful_exit(int retcode, bool prevent_crash_report) {
+	if (prevent_crash_report) {
+		/* Disable libFuzzer's atexit(). */
+		atexit_retcode = retcode;
+		atexit(&atexit_handler);
+	}
+	shutdown_lua();
+	exit(retcode);
+}
+
 NO_SANITIZE static int
 luaL_fuzz(lua_State *L)
 {
@@ -536,14 +563,8 @@ luaL_fuzz(lua_State *L)
 	jit_status = luajit_has_enabled_jit(L);
 #endif
 	set_global_lua_state(L);
-	int rc = LLVMFuzzerRunDriver(&argc, &argv, &TestOneInput);
-
-	free_argv(argc, argv);
-	luaL_cleanup(L);
-
-	lua_pushnumber(L, rc);
-
-	return 1;
+	graceful_exit(LLVMFuzzerRunDriver(&argc, &argv, &TestOneInput), true);
+	return 0;
 }
 
 static const struct luaL_Reg Module[] = {

luzer/tests/CMakeLists.txt

@@ -363,6 +363,11 @@ macro(generate_ffi_test name env_vars pass_regex)
   )
 endmacro()
 
+list(APPEND LSAN_REGULAR_EXPRESSION
+  "LeakSanitizer: detected memory leaks"
+  "[0-9]+ byte(s) leaked in [0-9]+ allocation"
+)
+
 list(APPEND TEST_ENV
   "LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR};"
 )
@@ -376,10 +381,7 @@ if (LUA_HAS_JIT)
     LD_PRELOAD=${ASAN_DSO_PATH}
     FFI_LIB_NAME=testlib_asan${CMAKE_SHARED_LIBRARY_SUFFIX}
   )
-  # XXX: Memory leak in FDP is expected on Linux, should be fixed
-  # in [1].
-  # 1. https://github.com/ligurio/luzer/issues/52
-  set(PASS_PATTERN "LeakSanitizer: detected memory leaks")
+  set(PASS_PATTERN ${LSAN_REGULAR_EXPRESSION})
   if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     set(PASS_PATTERN "Done 10 runs in 0 second")
   endif()
@@ -419,3 +421,33 @@ if (LUA_HAS_JIT)
     "runtime error: load of null pointer of type"
   )
 endif()
+
+string(JOIN ";" TEST_ENV
+  "LUA_CPATH=${LUA_CPATH}"
+  "LUA_PATH=${LUA_PATH}"
+  "LD_PRELOAD=${ASAN_DSO_PATH}"
+)
+add_test(
+  NAME luzer_missed_dict_test
+  COMMAND ${LUA_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_options_3.lua
+          -dict=${CMAKE_CURRENT_SOURCE_DIR}/nonexistent.dict
+)
+set_tests_properties(luzer_missed_dict_test PROPERTIES
+  ENVIRONMENT "${TEST_ENV}"
+  PASS_REGULAR_EXPRESSION "ParseDictionaryFile: file does not exist or is empty"
+  FAIL_REGULAR_EXPRESSION "${LSAN_REGULAR_EXPRESSION}"
+)
+
+add_test(
+  NAME luzer_leak_memory_test
+  COMMAND ${LUA_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/leak_memory.lua
+          -rss_limit_mb=500
+)
+list(APPEND PASS_REGULAR_EXPRESSION
+  "libFuzzer disabled leak detection after every mutation"
+  "ERROR: libFuzzer: out-of-memory"
+)
+set_tests_properties(luzer_leak_memory_test PROPERTIES
+  ENVIRONMENT "${TEST_ENV}"
+  PASS_REGULAR_EXPRESSION "${PASS_REGULAR_EXPRESSION}"
+)

luzer/tests/leak_memory.lua

@@ -0,0 +1,21 @@
+local luzer = require("luzer")
+
+local leaky_cache = {} -- luacheck: no unused
+
+-- Function to create a large, nested table.
+local function make_big_object(size)
+    if size <= 0 then
+        return {}
+    end
+    -- Create nested tables recursively.
+    return {
+        make_big_object(size - 1)
+    }
+end
+
+local function TestOneInput(_buf)
+    local new_object = make_big_object(2)
+    table.insert(leaky_cache, new_object)
+end
+
+luzer.Fuzz(TestOneInput)

luzer/tests/test_options_3.lua

@@ -0,0 +1,10 @@
+local luzer = require("luzer")
+
+local function TestOneInput(buf)
+    local fdp = luzer.FuzzedDataProvider(buf)
+    local str = fdp:consume_string(100)
+    local str_chars = {}
+    str:gsub(".", function(c) table.insert(str_chars, c) end)
+end
+
+luzer.Fuzz(TestOneInput)