Merging some WASIp3 PRs together, some more refactorings by me

cmake/bindings.cmake

@@ -107,6 +107,31 @@ add_custom_target(
       --rename wasi:cli/terminal-stdin@${wasip3-version}=terminal_stdin
       --rename wasi:cli/terminal-stdout@${wasip3-version}=terminal_stdout
       --rename wasi:cli/terminal-stderr@${wasip3-version}=terminal_stderr
+
+      # Disable async bindings generation for some functions which are only
+      # ever called synchronously within libc.
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.metadata-hash"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.metadata-hash-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.stat"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.stat-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.get-flags"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.open-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.read-directory"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.create-directory-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.remove-directory-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.unlink-file-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.advise"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.sync-data"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.sync"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.set-size"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.symlink-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.link-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.readlink-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.rename-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.set-times-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.set-times"
+      "--async=-wasi:clocks/monotonic-clock@${wasip3-version}#wait-until"
+      "--async=-wasi:clocks/monotonic-clock@${wasip3-version}#wait-for"
       ${CMAKE_SOURCE_DIR}/wasi/p3/wit
   COMMAND cmake -E copy wasip3.h ${bottom_half}/headers/public/wasi/__generated_wasip3.h
   COMMAND cmake -E copy wasip3_component_type.o ${bottom_half}/sources

expected/wasm32-wasip2/defined-symbols.txt

@@ -1494,6 +1494,7 @@ vwprintf
 vwscanf
 wall_clock_now
 wall_clock_resolution
+wasi_string_from_c
 wasip2_list_string_free
 wasip2_list_tuple2_string_string_free
 wasip2_list_u32_free
@@ -1502,7 +1503,6 @@ wasip2_option_string_free
 wasip2_string_dup
 wasip2_string_dup_n
 wasip2_string_free
-wasip2_string_from_c
 wasip2_string_set
 wasip2_tuple2_string_string_free
 wcpcpy

expected/wasm32-wasip3/defined-symbols.txt

@@ -331,7 +331,7 @@ __wasilibc_parse_port
 __wasilibc_populate_preopens
 __wasilibc_pthread_self
 __wasilibc_random
-__wasilibc_read_stream3
+__wasilibc_read
 __wasilibc_rename_newat
 __wasilibc_rename_oldat
 __wasilibc_reset_preopens
@@ -345,7 +345,7 @@ __wasilibc_unspecified_addr
 __wasilibc_utimens
 __wasilibc_wasi_family_to_libc
 __wasilibc_wasi_to_sockaddr
-__wasilibc_write_stream3
+__wasilibc_write
 __wasm_call_dtors
 __wcscoll_l
 __wcsftime_l
@@ -1463,21 +1463,25 @@ vswprintf
 vswscanf
 vwprintf
 vwscanf
+wasi_string_from_c
 wasip3_backpressure_dec
 wasip3_backpressure_inc
 wasip3_context_get_0
 wasip3_context_get_1
 wasip3_context_set_0
 wasip3_context_set_1
+wasip3_future_block_on
 wasip3_list_string_free
 wasip3_list_tuple2_string_string_free
 wasip3_list_u8_free
 wasip3_option_string_free
+wasip3_stream_block_on
 wasip3_string_dup
 wasip3_string_dup_n
 wasip3_string_free
 wasip3_string_set
 wasip3_subtask_block_on
+wasip3_subtask_block_on_and_drop
 wasip3_subtask_cancel
 wasip3_subtask_drop
 wasip3_task_cancel
@@ -1495,7 +1499,6 @@ wasip3_thread_yield_cancellable
 wasip3_thread_yield_to_suspended
 wasip3_thread_yield_to_suspended_cancellable
 wasip3_tuple2_string_string_free
-wasip3_waitable_block_on
 wasip3_waitable_join
 wasip3_waitable_set_drop
 wasip3_waitable_set_new

libc-bottom-half/CMakeLists.txt

@@ -139,6 +139,8 @@ else()
     sources/descriptor_table.c
     sources/getsockpeername.c
     sources/listen.c
+    sources/file_utils.c
+    sources/file.c
     sources/recv.c
     sources/send.c
     sources/shutdown.c
@@ -154,8 +156,6 @@ if(WASI STREQUAL "p2")
   list(APPEND bottom_half_sources
     sources/netdb.c
     sources/wasip2.c
-    sources/wasip2_file.c
-    sources/wasip2_file_utils.c
     sources/wasip2_stdio.c
   )
 endif()
@@ -164,8 +164,6 @@ if (WASI STREQUAL "p3")
   list(APPEND bottom_half_sources
     sources/wasip3.c
     sources/wasip3_block_on.c
-    sources/wasip3_file.c
-    sources/wasip3_file_utils.c
     sources/wasip3_stdio.c
   )
 endif()

libc-bottom-half/cloudlibc/src/libc/dirent/dirent_impl.h

@@ -28,7 +28,8 @@ struct _DIR {
   size_t skip;
   size_t offset;
 #elif defined(__wasip3__)
-  filesystem_stream_directory_entry_t stream;
+  filesystem_tuple2_stream_directory_entry_future_result_void_error_code_t stream;
+  bool stream_done;
   size_t skip;
   size_t offset;
 #else
@@ -40,4 +41,27 @@ struct _DIR {
   size_t dirent_size;
 };
 
+static inline void dirent_close_streams(DIR *dirp) {
+#if defined(__wasip1__)
+  // nothing to close ...
+#elif defined(__wasip2__)
+  if (dirp->stream.__handle) {
+    filesystem_directory_entry_stream_drop_own(dirp->stream);
+    dirp->stream.__handle = 0;
+  }
+#elif defined(__wasip3__)
+  if (dirp->stream.f0 != 0) {
+    filesystem_stream_directory_entry_drop_readable(dirp->stream.f0);
+    dirp->stream.f0 = 0;
+    dirp->stream_done = false;
+  }
+  if (dirp->stream.f1 != 0) {
+    filesystem_future_result_void_error_code_drop_readable(dirp->stream.f1);
+    dirp->stream.f1 = 0;
+  }
+#else
+# error "Unknown WASI version"
+#endif
+}
+
 #endif

libc-bottom-half/cloudlibc/src/libc/dirent/fdclosedir.c

@@ -9,17 +9,10 @@
 
 int fdclosedir(DIR *dirp) {
   int fd = dirp->fd;
-#if defined(__wasip1__)
+#ifdef __wasip1__
   free(dirp->buffer);
-#elif defined(__wasip2__)
-  if (dirp->stream.__handle != 0)
-    filesystem_directory_entry_stream_drop_own(dirp->stream);
-#elif defined(__wasip3__)
-  if (dirp->stream != 0)
-    filesystem_stream_directory_entry_drop_readable(dirp->stream);
-#else
-# error "Unsupported WASI version"
 #endif
+  dirent_close_streams(dirp);
   free(dirp->dirent);
   free(dirp);
   return fd;

libc-bottom-half/cloudlibc/src/libc/dirent/fdopendir.c

@@ -49,7 +49,7 @@ DIR *fdopendir(int fd) {
   dirp->dirent = NULL;
   dirp->dirent_size = 1;
   return dirp;
-#elif defined(__wasip2__)
+#elif defined(__wasip2__) || defined(__wasip3__)
   // Translate the file descriptor to an internal handle
   filesystem_borrow_descriptor_t file_handle;
   if (fd_to_file_handle(fd, &file_handle) < 0) {
@@ -58,6 +58,7 @@ DIR *fdopendir(int fd) {
   }
 
   // Read the directory
+#if defined(__wasip2__)
   filesystem_own_directory_entry_stream_t result;
   filesystem_error_code_t error_code;
   bool ok = filesystem_method_descriptor_read_directory(file_handle,
@@ -69,19 +70,17 @@ DIR *fdopendir(int fd) {
     return NULL;
   }
 
-  dirp->fd = fd;
   dirp->stream = result;
+#elif defined(__wasip3__)
+  filesystem_method_descriptor_read_directory(file_handle, &dirp->stream);
+  dirp->stream_done = false;
+#endif
+  dirp->fd = fd;
   dirp->skip = 0;
   dirp->offset = 0;
   dirp->dirent = NULL;
   dirp->dirent_size = 1;
   return dirp;
-#elif defined(__wasip3__)
-  (void) fd;
-  // TODO(wasip3)
-  errno = ENOTSUP;
-  free(dirp);
-  return NULL;
 #else
 # error "Unsupported WASI version"
 #endif

libc-bottom-half/cloudlibc/src/libc/dirent/readdir.c

@@ -31,8 +31,11 @@ static_assert(DT_REG == __WASI_FILETYPE_REGULAR_FILE, "Value mismatch");
 static_assert(DT_UNKNOWN == __WASI_FILETYPE_UNKNOWN, "Value mismatch");
 #endif
 
+#ifdef __wasip3__
+#include <wasi/wasip3_block.h>
+#endif
+
 // Grows a buffer to be large enough to hold a certain amount of data.
-#ifndef __wasip3__
 static struct dirent* grow(struct dirent **buffer, size_t *buffer_size, size_t target_size) {
   if (*buffer_size < target_size) {
     size_t new_size = *buffer_size;
@@ -48,7 +51,6 @@ static struct dirent* grow(struct dirent **buffer, size_t *buffer_size, size_t t
   }
   return *buffer;
 }
-#endif
 
 #if defined(__wasip1__)
 struct dirent *readdir(DIR *dirp) {
@@ -144,12 +146,13 @@ struct dirent *readdir(DIR *dirp) {
   }
 }
 
-#elif defined(__wasip2__)
+#elif defined(__wasip2__) || defined(__wasip3__)
 
 static int ensure_has_directory_stream(DIR *dirp, filesystem_borrow_descriptor_t *handle) {
   if (fd_to_file_handle(dirp->fd, handle) < 0)
     return -1;
 
+#ifdef __wasip2__
   if (dirp->stream.__handle != 0)
     return 0;
 
@@ -161,10 +164,15 @@ static int ensure_has_directory_stream(DIR *dirp, filesystem_borrow_descriptor_t
     translate_error(error_code);
     return -1;
   }
+#elif defined(__wasip3__)
+  if (dirp->stream.f0 == 0)
+    filesystem_method_descriptor_read_directory(*handle, &dirp->stream);
+#endif
   return 0;
 }
 
 static struct dirent *readdir_next(DIR *dirp) {
+  bool ok;
   filesystem_metadata_hash_value_t metadata;
   filesystem_error_code_t error_code;
   filesystem_borrow_descriptor_t dir_handle;
@@ -178,9 +186,9 @@ static struct dirent *readdir_next(DIR *dirp) {
     dirp->offset += 1;
     if (grow(&dirp->dirent, &dirp->dirent_size, offsetof(struct dirent, d_name) + 2) == NULL)
       return NULL;
-    bool ok = filesystem_method_descriptor_metadata_hash(dir_handle,
-                                                         &metadata,
-                                                         &error_code);
+    ok = filesystem_method_descriptor_metadata_hash(dir_handle,
+                                                    &metadata,
+                                                    &error_code);
     if (!ok) {
       translate_error(error_code);
       return NULL;
@@ -206,11 +214,12 @@ static struct dirent *readdir_next(DIR *dirp) {
     return dirp->dirent;
   }
 
+#if defined(__wasip2__)
   filesystem_borrow_directory_entry_stream_t stream = filesystem_borrow_directory_entry_stream(dirp->stream);
   filesystem_option_directory_entry_t dir_entry_optional;
-  bool ok = filesystem_method_directory_entry_stream_read_directory_entry(stream,
-                                                                          &dir_entry_optional,
-                                                                          &error_code);
+  ok = filesystem_method_directory_entry_stream_read_directory_entry(stream,
+                                                                     &dir_entry_optional,
+                                                                     &error_code);
   if (!ok) {
     translate_error(error_code);
     return NULL;
@@ -222,10 +231,57 @@ static struct dirent *readdir_next(DIR *dirp) {
 
   filesystem_directory_entry_t dir_entry = dir_entry_optional.val;
 
+#elif defined(__wasip3__)
+  filesystem_directory_entry_t dir_entry;
+
+  // Don't try to keep reading once the stream is closed.
+  if (dirp->stream_done)
+    return NULL;
+
+  // Loop until at least one stream entry is read, or until the stream is closed.
+  while (!dirp->stream_done) {
+    size_t amount =
+      wasip3_stream_block_on(
+          filesystem_stream_directory_entry_read(dirp->stream.f0, &dir_entry, 1),
+          dirp->stream.f0,
+          &dirp->stream_done);
+
+    // If something was read, then break out and process that below.
+    if (amount > 0)
+      break;
+
+    // If nothing was read and the stream isn't finished yet, try again.
+    if (!dirp->stream_done)
+      continue;
+
+    // If the stream's result future hasn't been read yet, do so here.
+    if (dirp->stream.f1) {
+      filesystem_result_void_error_code_t result;
+      wasip3_future_block_on(
+          filesystem_future_result_void_error_code_read(dirp->stream.f1, &result),
+          dirp->stream.f1);
+      filesystem_future_result_void_error_code_drop_readable(dirp->stream.f1);
+      dirp->stream.f1 = 0;
+      if (result.is_err)
+        translate_error(result.val.err);
+    }
+
+    // The stream is closed, so return NULL. If `errno` needs to be set it'll
+    // have been done above with `f1`.
+    return NULL;
+  }
+#else
+#error "Unknown WASI version"
+#endif
+
   // Ensure that the dirent is large enough to fit the filename
   size_t the_size = offsetof(struct dirent, d_name);
   if (grow(&dirp->dirent, &dirp->dirent_size, the_size + dir_entry.name.len + 1) == NULL) {
+#ifdef __wasip2__
     wasip2_string_free(&dir_entry.name);
+#else
+    wasip3_string_free(&dir_entry.name);
+#endif
     return NULL;
   }
 
@@ -241,7 +297,11 @@ static struct dirent *readdir_next(DIR *dirp) {
                                                      &dir_entry.name,
                                                      &metadata,
                                                      &error_code);
+#ifdef __wasip2__
   wasip2_string_free(&dir_entry.name);
+#else
+  wasip3_string_free(&dir_entry.name);
+#endif
   if (!ok) {
     translate_error(error_code);
     return NULL;
@@ -260,13 +320,6 @@ struct dirent *readdir(DIR *dirp) {
   }
   return result;
 }
-#elif defined(__wasip3__)
-struct dirent *readdir(DIR *dirp) {
-  // TODO(wasip3)
-  errno = ENOTSUP;
-  free(dirp);
-  return NULL;
-}
 #else
 # error "Unknown WASI version"
 #endif