Blake3 migration

.wakemanifest

@@ -58,7 +58,7 @@ tools/wake-hash/hash.wake
 tools/wake-migrate/wake-migrate.wake
 tools/wake-unit/wake-unit.wake
 tools/wake/wake.wake
-vendor/blake2/blake2.wake
+vendor/blake3/blake3.wake
 vendor/emscripten.wake
 vendor/fuse.wake
 vendor/git.wake

LICENSE

@@ -4,7 +4,7 @@ Each file includes a header which describes the relevant license.
 In broad strokes:
   wake          LICENSE.Apache2   everything not in vendor/
 
-  BLAKE2                    LICENSE.CC0       vendor/blake2/
+  BLAKE3                    LICENSE.CC0       vendor/blake3/
   gopt                      LICENSE.TFL       vendor/gopt/
   lemon                     public domain     vendor/lemon/
   SipHash                   LICENSE.CC0       vendor/siphash/

Makefile

@@ -28,14 +28,45 @@ COMMON_CPP  := $(foreach dir,$(COMMON_DIRS),$(wildcard $(dir)/*.cpp))
 COMMON_OBJS := src/json/jlexer.o \
                $(patsubst %.cpp,%.o,$(COMMON_CPP)) $(patsubst %.c,%.o,$(COMMON_C))
 
+# BLAKE3 platform detection
+UNAME_M := $(shell uname -m)
+
+# BLAKE3 C files (always compiled)
+BLAKE3_C_COMMON := vendor/blake3/blake3.c vendor/blake3/blake3_portable.c vendor/blake3/blake3_dispatch.c
+
+# BLAKE3 platform-specific files and flags
+ifeq ($(UNAME_M),x86_64)
+    # x86_64: Use SSE2/SSE4.1/AVX2/AVX-512 assembly implementations
+    BLAKE3_ASM := vendor/blake3/blake3_sse2_x86-64_unix.S \
+                  vendor/blake3/blake3_sse41_x86-64_unix.S \
+                  vendor/blake3/blake3_avx2_x86-64_unix.S \
+                  vendor/blake3/blake3_avx512_x86-64_unix.S
+    BLAKE3_C := $(BLAKE3_C_COMMON)
+    BLAKE3_CFLAGS :=
+else ifeq ($(UNAME_M),aarch64)
+    # ARM64: Use NEON implementation (requires BLAKE3_USE_NEON=1 for dispatch)
+    BLAKE3_ASM :=
+    BLAKE3_C := $(BLAKE3_C_COMMON) vendor/blake3/blake3_neon.c
+    BLAKE3_CFLAGS := -DBLAKE3_USE_NEON=1
+else
+    # Other platforms: Use portable implementation only
+    BLAKE3_ASM :=
+    BLAKE3_C := $(BLAKE3_C_COMMON)
+    BLAKE3_CFLAGS :=
+endif
+
+BLAKE3_ASM_OBJS := $(patsubst %.S,%.o,$(BLAKE3_ASM))
+BLAKE3_OBJS := $(patsubst %.c,%.o,$(BLAKE3_C)) $(BLAKE3_ASM_OBJS)
+
 WAKE_DIRS := $(COMMON_DIRS) src/dst src/optimizer src/parser src/runtime src/types src/wcl tools/wake
+
 WAKE_C    := $(foreach dir,$(WAKE_DIRS),$(wildcard $(dir)/*.c)) \
-             vendor/blake2/blake2b-ref.c vendor/utf8proc/utf8proc.c \
+             $(BLAKE3_C) vendor/utf8proc/utf8proc.c \
              vendor/siphash/siphash.c vendor/whereami/whereami.c \
              vendor/gopt/gopt.c vendor/gopt/gopt-errors.c vendor/gopt/gopt-arg.c
 WAKE_CPP  := $(foreach dir,$(WAKE_DIRS),$(wildcard $(dir)/*.cpp))
 WAKE_OBJS := src/parser/lexer.o src/parser/parser.o src/json/jlexer.o \
-             $(patsubst %.cpp,%.o,$(WAKE_CPP)) $(patsubst %.c,%.o,$(WAKE_C))
+             $(patsubst %.cpp,%.o,$(WAKE_CPP)) $(patsubst %.c,%.o,$(WAKE_C)) $(BLAKE3_ASM_OBJS)
 
 WAKE_ENV := WAKE_PATH=$(shell dirname $(shell which $(firstword $(CC))))
 
@@ -104,10 +135,10 @@ bin/wakebox:		tools/wakebox/main.cpp src/wakefs/*.cpp vendor/gopt/*.c $(COMMON_O
 lib/wake/fuse-waked:	tools/fuse-waked/main.cpp $(COMMON_OBJS)
 	$(CXX) $(CFLAGS) $(LOCAL_CFLAGS) $(FUSE_CFLAGS) $(CXX_VERSION) $^ -o $@ $(LDFLAGS)  $(CORE_LDFLAGS) $(FUSE_LDFLAGS)
 
-lib/wake/shim-wake:	tools/shim-wake/main.o vendor/blake2/blake2b-ref.o src/wcl/filepath.o
+lib/wake/shim-wake:	tools/shim-wake/main.o $(BLAKE3_OBJS) src/wcl/filepath.o
 	$(CXX) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(CORE_LDFLAGS)
 
-lib/wake/wake-hash: tools/wake-hash/main.o vendor/blake2/blake2b-ref.o $(COMMON_OBJS)
+lib/wake/wake-hash: tools/wake-hash/main.o $(BLAKE3_OBJS) $(COMMON_OBJS)
 	$(CXX) $(CFLAGS) -o $@ $^ $(LOCAL_CFLAGS) $(CXX_VERSION) $(LDFLAGS) $(CORE_LDFLAGS)
 
 bin/wake-migrate: tools/wake-migrate/main.o $(COMMON_OBJS)
@@ -119,6 +150,14 @@ bin/wake-migrate: tools/wake-migrate/main.o $(COMMON_OBJS)
 %.o:	%.c	$(filter-out src/version.h,$(wildcard */*.h))
 	$(CC) $(CFLAGS) $(LOCAL_CFLAGS) -o $@ -c $<
 
+# C rule for BLAKE3 files (needs BLAKE3_CFLAGS for NEON on aarch64)
+vendor/blake3/%.o:	vendor/blake3/%.c
+	$(CC) $(CFLAGS) $(LOCAL_CFLAGS) $(BLAKE3_CFLAGS) -o $@ -c $<
+
+# Assembly rule for BLAKE3 SIMD implementations
+vendor/blake3/%.o:	vendor/blake3/%.S
+	$(CC) $(CFLAGS) -o $@ -c $<
+
 # Rely on wake to recreate this file if re2c is available
 %.cpp:	%.cpp.gz
 	gzip -dc $^ > $@.tmp

debian/copyright

@@ -15,9 +15,9 @@ Copyright: 2012-2016 Jean-Philippe Aumasson <jeanphilippe.aumasson@gmail.com>
            2012-2014 Daniel J. Bernstein <djb@cr.yp.to>
 License: CC0
 
-Files: shim/blake2*
-Copyright: 2012 Samuel Neves <sneves@dei.uc.pt>
-License: CC0
+Files: vendor/blake3/*
+Copyright: 2019-2020 Samuel Neves and Jack O'Connor
+License: CC0 or Apache-2.0
 
 Files: utf8proc/*
 Copyright: 2018 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.

src/runtime/runtime.wake

@@ -22,5 +22,5 @@ target runtime variant =
     @here
     variant
     (optimizer, dst, types, json, util, Nil)
-    (util, gmp, sqlite3, re2, ncurses, blake2, Nil)
+    (util, gmp, sqlite3, re2, ncurses, blake3, Nil)
     Nil

src/runtime/schema.h

@@ -1,7 +1,7 @@
 #ifndef WAKE_SCHEMA_H
 #define WAKE_SCHEMA_H
 
-#define SCHEMA_VERSION "9"
+#define SCHEMA_VERSION "10"
 
 // Increment the SCHEMA_VERSION every time the below string changes.
 // Also add migrations to the wake-migration tool if needed.

src/runtime/string.cpp

@@ -29,7 +29,7 @@
 #include <fstream>
 #include <sstream>
 
-#include "blake2/blake2.h"
+#include "blake3/blake3.h"
 #include "gc.h"
 #include "json/utf8.h"
 #include "prim.h"
@@ -756,19 +756,19 @@ static PRIMTYPE(type_hash_str) {
   return args.size() == 1 && args[0]->unify(Data::typeString) && out->unify(Data::typeString);
 }
 
-static void blake2b(const std::string &str, uint64_t (*out)[4]) {
-  blake2b_state S;
-  blake2b_init(&S, sizeof(*out));
-  blake2b_update(&S, reinterpret_cast<const uint8_t *>(str.data()), str.size());
-  blake2b_final(&S, reinterpret_cast<uint8_t *>(*out), sizeof(*out));
+static void blake3(const std::string &str, uint64_t (*out)[4]) {
+  blake3_hasher hasher;
+  blake3_hasher_init(&hasher);
+  blake3_hasher_update(&hasher, reinterpret_cast<const uint8_t *>(str.data()), str.size());
+  blake3_hasher_finalize(&hasher, reinterpret_cast<uint8_t *>(*out), sizeof(*out));
 }
 
 static PRIMFN(prim_hash_str) {
   EXPECT(1);
   STRING(str, 0);
 
   uint64_t hash[4];
-  blake2b(str->as_str(), &hash);
+  blake3(str->as_str(), &hash);
   auto hex = wcl::to_hex(&hash);
 
   RETURN(String::alloc(runtime.heap, std::move(hex)));

tools/shim-wake/main.cpp

@@ -31,7 +31,7 @@
 #include <sys/time.h>
 #include <unistd.h>
 
-#include "blake2/blake2.h"
+#include "blake3/blake3.h"
 #include "compat/nofollow.h"
 #include "wcl/filepath.h"
 
@@ -45,7 +45,7 @@ static int do_hash_dir() {
 }
 
 static int do_hash_link(const char *link) {
-  blake2b_state S;
+  blake3_hasher hasher;
   uint8_t hash[HASH_BYTES];
   constexpr ssize_t len = 8192;
   char buffer[len];
@@ -61,9 +61,9 @@ static int do_hash_link(const char *link) {
     return 1;
   }
 
-  blake2b_init(&S, sizeof(hash));
-  blake2b_update(&S, (uint8_t *)buffer, out);
-  blake2b_final(&S, &hash[0], sizeof(hash));
+  blake3_hasher_init(&hasher);
+  blake3_hasher_update(&hasher, (uint8_t *)buffer, out);
+  blake3_hasher_finalize(&hasher, &hash[0], sizeof(hash));
 
   for (int i = 0; i < (int)sizeof(hash); ++i) printf("%02x", hash[i]);
   printf("\n");
@@ -72,13 +72,14 @@ static int do_hash_link(const char *link) {
 }
 
 static int do_hash_file(const char *file, int fd) {
-  blake2b_state S;
+  blake3_hasher hasher;
   uint8_t hash[HASH_BYTES], buffer[8192];
   ssize_t got;
 
-  blake2b_init(&S, sizeof(hash));
-  while ((got = read(fd, &buffer[0], sizeof(buffer))) > 0) blake2b_update(&S, &buffer[0], got);
-  blake2b_final(&S, &hash[0], sizeof(hash));
+  blake3_hasher_init(&hasher);
+  while ((got = read(fd, &buffer[0], sizeof(buffer))) > 0)
+    blake3_hasher_update(&hasher, &buffer[0], got);
+  blake3_hasher_finalize(&hasher, &hash[0], sizeof(hash));
 
   if (got < 0) {
     fprintf(stderr, "shim hash read(%s): %s\n", file, strerror(errno));

tools/shim-wake/shim.wake

@@ -19,4 +19,4 @@ from wake import _
 from gcc_wake import _
 
 target buildShim variant: Result (List Path) Error =
-    tool @here Nil variant "lib/wake/shim-wake" (blake2, compat, wcl, Nil) Nil Nil
+    tool @here Nil variant "lib/wake/shim-wake" (blake3, compat, wcl, Nil) Nil Nil

tools/wake-hash/hash.wake

@@ -19,4 +19,4 @@ from wake import _
 from gcc_wake import _
 
 target buildHash variant: Result (List Path) Error =
-    tool @here Nil variant "lib/wake/wake-hash" (blake2, compat, Nil) Nil Nil
+    tool @here Nil variant "lib/wake/wake-hash" (blake3, compat, Nil) Nil Nil

tools/wake-hash/main.cpp

@@ -40,7 +40,7 @@
 #include <thread>
 #include <vector>
 
-#include "blake2/blake2.h"
+#include "blake3/blake3.h"
 #include "compat/nofollow.h"
 #include "wcl/optional.h"
 #include "wcl/unique_fd.h"
@@ -130,7 +130,7 @@ static wcl::optional<Hash256> hash_exotic() {
 static wcl::optional<Hash256> hash_dir() { return wcl::some(Hash256()); }
 
 static wcl::optional<Hash256> hash_link(const char* link) {
-  blake2b_state S;
+  blake3_hasher hasher;
   uint8_t hash[HASH_BYTES];
   std::vector<char> buffer(8192, 0);
 
@@ -147,21 +147,22 @@ static wcl::optional<Hash256> hash_link(const char* link) {
     buffer.resize(2 * buffer.size(), 0);
   }
 
-  blake2b_init(&S, sizeof(hash));
-  blake2b_update(&S, reinterpret_cast<uint8_t*>(buffer.data()), buffer.size());
-  blake2b_final(&S, &hash[0], sizeof(hash));
+  blake3_hasher_init(&hasher);
+  blake3_hasher_update(&hasher, reinterpret_cast<uint8_t*>(buffer.data()), buffer.size());
+  blake3_hasher_finalize(&hasher, &hash[0], sizeof(hash));
 
   return wcl::some(Hash256::from_hash(&hash));
 }
 
 static wcl::optional<Hash256> hash_file(const char* file, int fd) {
-  blake2b_state S;
+  blake3_hasher hasher;
   uint8_t hash[HASH_BYTES], buffer[8192];
   ssize_t got;
 
-  blake2b_init(&S, sizeof(hash));
-  while ((got = read(fd, &buffer[0], sizeof(buffer))) > 0) blake2b_update(&S, &buffer[0], got);
-  blake2b_final(&S, &hash[0], sizeof(hash));
+  blake3_hasher_init(&hasher);
+  while ((got = read(fd, &buffer[0], sizeof(buffer))) > 0)
+    blake3_hasher_update(&hasher, &buffer[0], got);
+  blake3_hasher_finalize(&hasher, &hash[0], sizeof(hash));
 
   if (got < 0) {
     std::cerr << "wake-hash read(" << file << "): " << strerror(errno) << std::endl;

tools/wake-migrate/main.cpp

@@ -190,6 +190,22 @@ static std::vector<Migration> get_migrations() {
        },
        "Convert runner_status from INTEGER to TEXT"},
 
+      // Version 9 -> 10: Hash algorithm changed from BLAKE2b to BLAKE3
+      // Migration is not supported - user must delete wake.db and reinitialize
+      {9, 10,
+       [](sqlite3* db) -> bool {
+         (void)db;  // unused
+         std::cerr << std::endl;
+         std::cerr << "Wake has been upgraded from BLAKE2b to BLAKE3 hashing." << std::endl;
+         std::cerr << "Your existing wake.db is incompatible and must be deleted." << std::endl;
+         std::cerr << std::endl;
+         std::cerr << "Please run:" << std::endl;
+         std::cerr << "  rm wake.db && wake --init ." << std::endl;
+         std::cerr << std::endl;
+         return false;
+       },
+       "Hash algorithm changed from BLAKE2b to BLAKE3 - manual deletion required"},
+
   };
 }