Merge pull request #761 from permaweb/twilson63/feat-update-dev-cli-w…

…asm64

feat(dev-cli): update to wasm64

dev-cli/container/Dockerfile

@@ -1,4 +1,4 @@
-FROM emscripten/emsdk:3.1.55
+FROM emscripten/emsdk:3.1.59
 LABEL maintainer "tom wilson <[email protected]>"
 
 # The working directory used by the base image is /src, so we can mount volumes to there
@@ -90,6 +90,12 @@ COPY ./src/main.c /opt/main.c
 COPY ./src/main.lua /opt/main.lua
 RUN chmod +x /usr/local/bin/emcc-lua
 
+###################################
+# BUILD WeaveDrive Extension Helper
+###################################
+COPY ./src/aolibc /opt/aolibc
+RUN cd /opt/aolibc && make CC="emcc -s WASM=1 -s MEMORY64=1 -s SUPPORT_LONGJMP=1"
+
 ENV CC 'emcc -s WASM=1'
 ENV NM 'llvm-nm'
 

dev-cli/container/src/aolibc/Makefile

@@ -0,0 +1,45 @@
+# Builds a patched libc.a, customized with the override functions found in aostdlib.c
+
+# Path to the Emscripten SDK
+EMSDK_PATH ?= $(shell echo $$EMSDK)
+LIBC_A_PATH ?= $(shell find $(EMSDK_PATH) -name libc.a)
+PATCHED_LIBC_A_PATH ?= $(shell pwd)/libc.ao.a
+
+# Path to your custom stdlib object file
+CUSTOM_LIB_OBJ = aostdio.o
+
+.PHONY: build
+build: aolibc.a
+
+aolibc.a: $(CUSTOM_LIB_OBJ)
+	emar r aolibc.a $(CUSTOM_LIB_OBJ)
+
+CUSTOM_LIB_OBJ: aostdio.c
+	emcc -s -c aostdio.c -o aostdio.o
+
+# Default target
+install: build-patched update-libc
+
+# Target to update libc.a
+build-patched: $(CUSTOM_LIB_OBJ)
+	@# Ensure libc.a path is not empty
+	@if [ -z "$(LIBC_A_PATH)" ]; then \
+		echo "libc.a not found, check your EMSDK_PATH"; \
+		exit 1; \
+	fi
+	@cp $(LIBC_A_PATH) $(PATCHED_LIBC_A_PATH)
+	@echo "Updating libc.a at $(PATCHED_LIBC_A_PATH)..."
+	@# Extract function names from the custom object file
+	$(eval FUNCTIONS_TO_REMOVE=$(shell emnm $(CUSTOM_LIB_OBJ) | grep ' T ' | cut -d' ' -f3 | sed 's/$$/.o/'))
+	@echo "Removing functions: $(FUNCTIONS_TO_REMOVE)"
+	@# Remove existing function implementations from libc.a
+	emar d $(PATCHED_LIBC_A_PATH) $(FUNCTIONS_TO_REMOVE)
+	@# Add custom stdlib object file to libc.a
+	emar r $(PATCHED_LIBC_A_PATH) $(CUSTOM_LIB_OBJ)
+	@echo "libc.a updated successfully."
+
+# Clean up
+clean:
+	@echo "Cleaning up..."
+	@rm -f $(CUSTOM_LIB_OBJ)
+	@echo "Clean complete."

dev-cli/container/src/aolibc/aostdio.c

@@ -0,0 +1,88 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+#include <emscripten.h>
+
+#if 0
+#define AO_LOG(...) fprintf(stderr, __VA_ARGS__)
+#else
+#define AO_LOG(...)
+#endif
+
+// WeaveDrive async wrapper functions. These allow us to call the WeaveDrive
+// async JS code from C.
+EM_ASYNC_JS(int, weavedrive_open, (const char* c_filename, const char* mode), {
+    const filename = UTF8ToString(Number(c_filename));
+    if (!Module.WeaveDrive) {
+        return Promise.resolve(null)
+    }
+
+    const drive = Module.WeaveDrive(Module, FS);
+    return await drive.open(filename);
+});
+
+EM_ASYNC_JS(int, weavedrive_read, (int fd, int *dst_ptr, size_t length), {
+    const drive = Module.WeaveDrive(Module, FS);
+    return Promise.resolve(await drive.read(fd, dst_ptr, length));
+});
+
+FILE* fopen(const char* filename, const char* mode) {
+    AO_LOG( "AO: Called fopen: %s, %s\n", filename, mode);
+    int fd = weavedrive_open(filename, mode);
+    AO_LOG( "AO: weavedrive_open returned fd: %d\n", fd);
+    // If we get a file desciptor, we return a FILE*, else 0.
+    if (!fd) {
+        return 0;
+    }
+    return fdopen(fd, mode);
+}
+
+size_t fread(void* ptr, size_t size, size_t nmemb, FILE* stream) {
+    int fd = fileno(stream);
+    weavedrive_read(fd, ptr, size * nmemb);
+    return nmemb;
+}
+
+int fclose(FILE* stream) {
+     AO_LOG( "AO: fclose called\n");
+     return 0;  // Returning success, adjust as necessary
+}
+
+void* realloc(void* ptr, size_t size) {
+    void* new_ptr = memalign(16, size);
+    memcpy(new_ptr, ptr, size);
+    free(ptr);
+    //AO_LOG("DBG: Realloc called: %p -> %p, size: %zu\n", ptr, new_ptr, size);
+    return new_ptr;
+}
+
+// Emscripten malloc does not align to 16 bytes correctly, which causes some 
+// programs that use aligned memory (for example, those that use SIMD...) to
+// crash. So we need to use the aligned allocator.
+void* malloc(size_t size) {
+    return memalign(16, size);
+}
+
+int madvise(void* addr, size_t length, int advice) {
+    AO_LOG("AO: madvise called with addr: %p, length: %zu, advice: %d\n", addr, length, advice);
+    return 0;
+}
+
+void* mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) {
+    AO_LOG("AO: mmap called with addr: %p, length: %zu, prot: %d, flags: %d, fd: %d, offset: %d\n", addr, length, prot, flags, fd, offset);
+    // Allocate a buffer that fits with emscripten's normal allignments 
+    void* buffer = memalign(65536, length);
+    AO_LOG("AO: mmap: Reading from arweave to: %p, length: %zu\n", buffer, length);
+    weavedrive_read(fd, buffer, length);
+    AO_LOG("AO: mmap returned: %p\n", buffer);
+    return buffer;
+}
+
+/*
+int munmap(void* addr, size_t length) {
+    AO_LOG("AO: munmap called with addr: %p, length: %zu\n", addr, length);
+    return 0;
+}
+*/

dev-cli/container/src/emcc-lua

@@ -164,7 +164,9 @@ def main():
 
     # Finally, compile to wasm
     debug_print('Start to compile as WASM')
-    cmd = ['emcc', '-O0', 
+    cmd = ['emcc', '-O3', 
+      '-g2',
+      '-s', 'ASYNCIFY=1',
       '-s', 'MEMORY64=1',
       '-s', 'ALLOW_MEMORY_GROWTH=1', 
       '-s', 'INITIAL_MEMORY=6291456', 
@@ -174,21 +176,26 @@ def main():
       # '-s', 'FILESYSTEM=0', 
       '-s', 'DETERMINISTIC=1', 
       '-s', 'NODERAWFS=0', 
+      '-s', 'FORCE_FILESYSTEM=1',
+      '-msimd128',
       '--pre-js', '/opt/pre.js'
     ]
+    cmd.extend(['-L/opt/aolibc', '-l:aolibc.a'])
     cmd.extend(definition.get_extra_args())
     cmd.extend(['-I', quote('/lua-{}/src'.format(os.environ.get('LUA_VERSION')))])
     cmd.extend(['/tmp/compile.c', quote('/lua-{}/src/liblua.a'.format(os.environ.get('LUA_VERSION')))])
     cmd.extend([quote(v.filepath) for v in link_libraries])
     cmd.extend([quote(v.filepath) for v in link_libraries])
+
+    cmd.extend(['-s', 'EXPORTED_FUNCTIONS=["_malloc"]'])
     cmd.extend(['-lm', '-ldl', '-o', definition.get_output_file(), '-s', 'EXPORTED_RUNTIME_METHODS=["cwrap"]'])
 
     debug_print('Compile command is {}'.format(' '.join(cmd)))
     shell_exec(*cmd)
 
     # add metering library
-    meter_cmd = ['node', '/opt/node/apply-metering.cjs']
-    shell_exec(*meter_cmd)
+    # meter_cmd = ['node', '/opt/node/apply-metering.cjs']
+    # shell_exec(*meter_cmd)
     shell_exec(*['rm', '/src/process.js'])
 
 if __name__ == '__main__':