refactor(dev-cli): refactored container and started code to allow mor…

…e language additions #828

dev-cli/README.md

@@ -1,7 +1,7 @@
 # ao CLI
 
 The `ao` cli enables developers to create, run, build, and publish
-SmartWeaveContracts written in [Lua](https://www.lua.org/).
+SmartWeaveContracts written in [Lua](https://www.lua.org/) and [C/C++](https://cplusplus.com/).
 
 - Initialize a Lua SmartWeaveContract template
 - Run Lua in a Repl or run your Lua SmartWeaveContract
@@ -16,7 +16,7 @@ SmartWeaveContracts written in [Lua](https://www.lua.org/).
   - [Initialize a new Project](#initialize-a-new-project)
   - [Run a Lua Repl](#run-a-lua-repl)
   - [Execute a Lua file](#execute-a-lua-file)
-  - [Build Lua to Wasm](#build-lua-to-wasm)
+  - [Build Lua or C/C++ to Wasm](#build-lua-or-cc-to-wasm)
   - [Publish Wasm to Permaweb](#publish-wasm-to-permaweb)
   - [Help](#help)
 - [Testing example](#testing-example)
@@ -68,7 +68,14 @@ Then follow the instructions for adding the `ao` binary to your `PATH`. Use
 ### Initialize a new Project
 
 ```sh
+# Initialize a Lua project by default.
 ao init [myproject]
+
+# Explicitly Initialize a Lua project.
+ao init -l lua [myproject]
+
+# Initialize a C/C++ project.
+ao init -l cpp [myproject] 
 ```
 
 This will create a new directory, if needed, named `{myproject}`
@@ -89,7 +96,7 @@ This is great for testing Lua modules
 ao run [file.lua]
 ```
 
-### Build Lua to Wasm
+### Build Lua or C/C++ to Wasm
 
 ```sh
 ao build

dev-cli/container/Dockerfile

@@ -75,20 +75,17 @@ RUN cd /lua-${LUA_VERSION} && \
 ###############
 #### Build ####
 ###############
-COPY ./src/emcc-lua /usr/local/bin/emcc-lua
-COPY ./src/emcc_lua_lib /usr/local/emcc-lua/emcc_lua_lib
+COPY ./src/ao-build-module /usr/local/bin/ao-build-module
+COPY ./src/ao_module_lib /usr/local/ao-module/ao_module_lib
 
-COPY ./src/pre.js /opt/pre.js
-COPY ./src/definition.yml /opt/definition.yml
-COPY ./src/loader.lua /opt/loader.lua
+COPY ./src/core/ /opt/
 
 RUN mkdir -p /opt/src
 COPY ./src/json.lua /opt/src/json.lua
 COPY ./src/ao.lua /opt/src/ao.lua
 # COPY ./src/pack.lua /opt/pack.lua
-COPY ./src/main.c /opt/main.c
-COPY ./src/main.lua /opt/main.lua
-RUN chmod +x /usr/local/bin/emcc-lua
+
+RUN chmod +x /usr/local/bin/ao-build-module
 
 ###################################
 # BUILD WeaveDrive Extension Helper

dev-cli/container/src/ao-build-module

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import glob
+import shutil
+from shlex import quote
+
+sys.path.append('/usr/local/ao-module')
+
+from ao_module_lib.definition import Definition
+from ao_module_lib.helper import is_lua_source_file, is_c_source_file, is_c_header_file, shell_exec, debug_print
+
+from ao_module_lib.languages.c import inject_c_files
+from ao_module_lib.languages.lua import inject_lua_files, LUAROCKS_LOCAL_MODULE_DIR
+
+
+def determine_language():
+    lang = []
+    # Get all source files
+    files = glob.glob('/src/**/*', recursive=True)
+    # Loop through all files and determine the language
+    for file in files:
+        if is_lua_source_file(file) and 'lua' not in lang:
+            lang.append('lua')
+        elif (is_c_source_file(file) or is_c_header_file(file)) and 'c' not in lang:
+            lang.append('c')
+
+    # Check if multiple languages are detected
+    if len(lang) > 1:
+        raise Exception('Multiple languages detected in the module')
+    elif len(lang) == 0:
+        raise Exception('No language detected in the module')
+
+    # Return the language or default to lua
+    return lang[0] or 'lua'
+
+def main():
+    # Load the definition file
+    definition = Definition('/opt/definition.yml')
+
+    # Determine the language of the source files
+    language = determine_language()
+
+    # Read the main.c file into c_program
+    c_program = ''
+    with open('/opt/main.c', mode='r') as c:
+        c_program = c.read()
+
+    # Inject c files into c_program if language is c
+    c_source_files = []
+    if(language == 'c'):
+        c_program = inject_c_files(definition, c_program, c_source_files)
+
+    # Inject lua files into c_program always to load lua files and replace placeholders
+    link_libraries = []
+    c_program = inject_lua_files(definition, c_program, link_libraries)
+
+    #Generate compile target file
+    debug_print('Start to generate complie.c')
+
+    with open('/tmp/compile.cpp', mode='w') as build:
+        build.write(c_program)
+
+    # Compile the module to a WASM file using emcc
+    debug_print('Start to compile as WASM')
+
+    # Setup the compile command
+    cmd = ['emcc', '-O3', 
+      '-g2',
+      '-s', 'ASYNCIFY=1',
+      '-s', 'MEMORY64=1',
+      '-s', 'STACK_SIZE=41943040',
+      '-s', 'ASYNCIFY_STACK_SIZE=41943040',
+      '-s', 'ALLOW_MEMORY_GROWTH=1', 
+      '-s', 'INITIAL_MEMORY=52428800', 
+      '-s', 'MAXIMUM_MEMORY=524288000', 
+      '-s', 'WASM=1', 
+      '-s', 'MODULARIZE', 
+      # '-s', 'FILESYSTEM=0', 
+      '-s', 'DETERMINISTIC=1', 
+      '-s', 'NODERAWFS=0', 
+      '-s', 'FORCE_FILESYSTEM=1',
+      '-msimd128',
+      '--pre-js', '/opt/pre.js'
+    ]
+
+    # Link aolibc library
+    cmd.extend(['-L/opt/aolibc', '-l:aolibc.a'])
+    cmd.extend(['-s', 'ASSERTIONS=1'])
+    cmd.extend(definition.get_extra_args())
+
+    # Add the lua include path
+    cmd.extend(['-I', quote('/lua-{}/src'.format(os.environ.get('LUA_VERSION')))])
+
+    # Add all c source files if there are any
+    for f in c_source_files:
+        cmd.append(quote(f))
+
+    # Add the compile file and link libraries
+    cmd.extend(['/tmp/compile.cpp', quote('/lua-{}/src/liblua.a'.format(os.environ.get('LUA_VERSION')))])
+    cmd.extend([quote(v.filepath) for v in link_libraries])
+    cmd.extend([quote(v.filepath) for v in link_libraries])
+
+    # Export the main function and malloc for the runtime
+    cmd.extend(['-s', 'EXPORTED_FUNCTIONS=["_malloc", "_main"]'])
+
+    # Export the runtime methods cwrap
+    cmd.extend(['-lm', '-ldl', '-o', definition.get_output_file(), '-s', 'EXPORTED_RUNTIME_METHODS=["cwrap"]'])
+
+    debug_print('Compile command is {}'.format(' '.join(cmd)))
+    shell_exec(*cmd)
+
+    # add metering library
+    # meter_cmd = ['node', '/opt/node/apply-metering.cjs']
+    # shell_exec(*meter_cmd)
+    shell_exec(*['rm', os.path.join(os.getcwd(), 'process.js')])
+
+if __name__ == '__main__':
+    main()
+
+    # cleanup temporary module directory
+    if os.path.isdir(LUAROCKS_LOCAL_MODULE_DIR):
+        shutil.rmtree(LUAROCKS_LOCAL_MODULE_DIR)

dev-cli/container/src/emcc_lua_lib/definition.py → dev-cli/container/src/ao_module_lib/definition.py

@@ -142,10 +142,10 @@ def make_function_delarations(self):
 
     def make_c_function_delarations(self):
         template = '''
+        
 EMSCRIPTEN_KEEPALIVE
 {} {}({}) {{
-    Process process;
-    return process.handle({});
+    return process_handle({});
     {}
 }}
 '''

dev-cli/container/src/emcc_lua_lib/helper.py → dev-cli/container/src/ao_module_lib/helper.py

@@ -38,3 +38,14 @@ def debug_print(message):
     if IS_DEBUG:
         print(message)
 
+
+def __get_output(output, entry_file):
+    out_file = os.path.basename(entry_file)
+    if output.get('file'):
+        out_file = output.get('file')
+
+    tpl = os.path.splitext(out_file)[1]
+    if not tpl[1]:
+        tpl[1] = 'html'
+
+    return '{}.{}'.format(tpl[0], tpl[1])

dev-cli/container/src/ao_module_lib/languages/c.py

@@ -0,0 +1,18 @@
+import glob
+from ao_module_lib.definition import Definition
+
+def inject_c_files(definition: Definition, c_program: str, c_source_files: list):
+
+    c_header_files = []
+    c_source_files += glob.glob('/src/**/*.c', recursive=True)
+    c_source_files += glob.glob('/src/**/*.cpp', recursive=True)
+    c_header_files += glob.glob('/src/**/*.h', recursive=True)    
+    c_header_files += glob.glob('/src/**/*.hpp', recursive=True)
+
+    for header in c_header_files:
+        c_program = '#include "{}"\n'.format(header) + c_program
+
+
+    c_program = 'const char *process_handle(const char *arg_0, const char *arg_1);\n' + c_program
+    c_program = c_program.replace('__FUNCTION_DECLARATIONS__', definition.make_c_function_delarations())
+    return c_program

dev-cli/container/src/ao_module_lib/languages/lua.py

@@ -0,0 +1,139 @@
+import glob
+import os
+import re
+import subprocess
+
+from ao_module_lib.definition import Definition
+from ao_module_lib.file import LuaFile, ModuleFile, BundleFile
+from ao_module_lib.helper import is_lua_source_file, is_binary_library, encode_hex_literals, shell_exec, debug_print
+
+CC = os.environ.get('CC', 'cc')
+NM = os.environ.get('NM', 'nm')
+
+LUAROCKS_LOCAL_MODULE_DIR = '/src/modules' # os.path.abspath('/src/modules')
+LUAROCKS_LOCAL_MODULE_PREFIX_RE = re.compile(re.escape(LUAROCKS_LOCAL_MODULE_DIR) + '\/share\/lua\/\d+.\d+/')
+
+def __get_uname():
+    uname = ''
+    try:
+        uname, _ = shell_exec('uname', '-s')
+    except (subprocess.CalledProcessError):
+        uname = 'Unknown'
+
+    return uname
+
+
+def inject_lua_files(definition: Definition, c_program: str, link_libraries: list):
+
+    uname = __get_uname()
+
+    lua_files = []
+    library_files = []
+    dependency_libraries = []
+
+    # entry_file = definition.get_entry_file()
+    entry_file = '/opt/loader.lua'
+
+    if not is_lua_source_file(entry_file):
+        print('main file of {} must be lua script.'.format(entry_file))
+        return
+
+    definition.install_dependencies(LUAROCKS_LOCAL_MODULE_DIR)
+
+    local_include_dir = os.path.join(os.path.dirname(entry_file), 'src')
+    local_include_prefix_re = re.compile(re.escape(local_include_dir + '/'))
+
+    # Detect bundles
+    bundle_files = glob.glob('/src/**/*.lua', recursive=True)
+
+    # Optional dependencies
+    bundle_files += glob.glob(local_include_dir + '/**/*.lua', recursive=True)
+    bundle_files += glob.glob(local_include_dir + '/**/*.so', recursive=True)
+    bundle_files += glob.glob(LUAROCKS_LOCAL_MODULE_DIR +
+                              '/lib/lua/**/*.so', recursive=True)
+    bundle_files += glob.glob(LUAROCKS_LOCAL_MODULE_DIR +
+                              '/share/lua/**/*.lua', recursive=True)
+    debug_print('Start to factory and distinguish module files')
+
+    for bundle in bundle_files:
+        if is_lua_source_file(bundle):
+            basename = re.sub(LUAROCKS_LOCAL_MODULE_PREFIX_RE, '', bundle)
+            basename = re.sub(local_include_prefix_re, '', basename)
+            lua_files.append(LuaFile(bundle, basename))
+            continue
+
+        if is_binary_library(bundle):
+            try:
+                nm, _ = shell_exec(NM, bundle)
+                is_module = False
+                if re.search(r'T _?luaL_newstate', nm):
+                    if re.search(r'U _?dlopen', nm):
+                        if uname in ['Linux', 'SunOS', 'Darwin']:
+                            libdl_option = '-ldl'
+
+                else:
+                    for luaopen in re.finditer(r'[^dD] _?luaopen_([0-9a-zA-Z!"#\$%&\'\(\)\*\+,\-\.\/:;\<=\>\?@\[\]^_`\{\|\}~]+)', nm):
+                        debug_print('luaopen_{} function found. add to library in {}'.format(
+                            luaopen.group(1), bundle))
+                        library_files.append(
+                            ModuleFile(bundle, luaopen.group(1)))
+                        is_module = True
+
+                if is_module:
+                    link_libraries.append(BundleFile(bundle))
+                else:
+                    dependency_libraries.append(BundleFile(bundle))
+
+            except (subprocess.CalledProcessError):
+                print(NM + ' command failed')
+                return
+
+    debug_print('===== Bundle Lua files ======')
+    debug_print('\n'.join([v.filepath for v in lua_files]))
+    debug_print('===== Library files =====')
+    debug_print('\n'.join([v.filepath for v in library_files]))
+    debug_print('===== Link libraries =====')
+    debug_print('\n'.join([v.filepath for v in link_libraries]))
+    debug_print('===== Dependency libraries =====')
+    debug_print('\n'.join([v.filepath for v in dependency_libraries]))
+
+    with open('/opt/main.lua', mode='r') as lua:
+        lua_program = lua.read()
+
+    c_program = c_program.replace(
+        '__LUA_BASE__', encode_hex_literals(lua_program))
+    with open(entry_file, mode='r') as main_file:
+        p = main_file.read()
+        c_program = c_program.replace('__LUA_MAIN__', encode_hex_literals(p))
+
+    inject_lua_files = []
+    for i, f in enumerate(lua_files):
+        with open(f.filepath, mode='r') as l:
+            lines = l.readlines()
+            # check first line has control command
+            if lines[0].find("\xef\xbb\xbf") != -1:
+                # UTF-8 byte order mark
+                lines[0] = lines[0][4:]
+            elif lines[0][0] == '#':
+                # shebang
+                lines = lines[1:]
+
+            inject_lua_files.extend([
+                '  static const unsigned char lua_require_{}[] = {{{}}};'.format(
+                    i, encode_hex_literals('\n'.join(lines))),
+                '  lua_pushlstring(L, (const char*)lua_require_{}, sizeof(lua_require_{}));'.format(i, i),
+                '  lua_setfield(L, -2, "{}");\n'.format(f.module_name)
+            ])
+
+    for f in library_files:
+        inject_lua_files.extend([
+            '  int luaopen_{}(lua_State* L);'.format(f.module_name),
+            '  lua_pushcfunction(L, luaopen_{});'.format(f.module_name),
+            '  lua_setfield(L, -2, "{}");'.format(f.basename)
+        ])
+    c_program = c_program.replace(
+        '__FUNCTION_DECLARATIONS__', definition.make_function_delarations())
+    c_program = c_program.replace(
+        '__INJECT_LUA_FILES__', '\n'.join(inject_lua_files))
+
+    return c_program

dev-cli/container/src/main.c → dev-cli/container/src/core/main.c

@@ -1,4 +1,3 @@
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <signal.h>