feat: build caching and library management

dev-cli/container/Dockerfile

@@ -6,8 +6,8 @@ LABEL maintainer="tom wilson <[email protected]>"
 # 
 # https://github.com/emscripten-core/emsdk/blob/9b0db91883452051aca8deddc932363aab29060b/docker/Dockerfile#L120
 
-RUN apt-get update -qq -y
-RUN apt-get install -y curl vim make gcc libreadline6-dev libssl-dev zlib1g-dev zip unzip
+RUN apt-get update --fix-missing -qq -y
+RUN apt-get install -y curl vim make gcc libreadline6-dev libssl-dev zlib1g-dev zip unzip ccache
 
 ENV LUA_VERSION=5.3.4
 ENV LUAROCKS_VERSION=2.4.4

dev-cli/container/src/ao-build-module

@@ -14,30 +14,39 @@ from ao_module_lib.helper import is_lua_source_file, is_c_source_file, is_c_head
 
 from ao_module_lib.languages.c import inject_c_files
 from ao_module_lib.languages.rust import inject_rust_files
-from ao_module_lib.languages.lua import inject_lua_files, LUAROCKS_LOCAL_MODULE_DIR
+from ao_module_lib.languages.lua import inject_lua_files
+from ao_module_lib.libraries import load_libraries, LUAROCKS_LOCAL_MODULE_DIR
 
 
 def determine_language():
-    lang = []
+    lang = 'lua'
+    langs = []
     # Get all source files
     files = glob.glob('/src/**/*', recursive=True)
     # Loop through all files and determine the language
     for file in files:
-        if is_lua_source_file(file) and 'lua' not in lang:
-            lang.append('lua')
-        elif (is_c_source_file(file)) and 'c' not in lang:
-            lang.append('c')
-        elif is_rust_source_file(file) and 'rust' not in lang:
-            lang.append('rust')
+        if is_lua_source_file(file) and 'lua' not in langs:
+            langs.append('lua')
+        elif (is_c_source_file(file)) and 'c' not in langs:
+            langs.append('c')
+        elif is_rust_source_file(file) and 'rust' not in langs:
+            langs.append('rust')
 
     # Check if multiple languages are detected
-    if len(lang) > 1:
-        raise Exception('Multiple languages detected in the module')
-    elif len(lang) == 0:
+    # if len(lang) > 1:
+    #     raise Exception('Multiple languages detected in the module')
+    if len(langs) == 0:
         raise Exception('No language detected in the module')
+
+    if langs.count('lua') == 0 and langs.count('c') >= 1:
+        lang = 'c'
+    elif langs.count('lua') == 0 and langs.count('rust') >= 1:
+        lang = 'rust'
+    else:
+        lang = 'lua'
 
     # Return the language or default to lua
-    return lang[0] or 'lua'
+    return lang
 
 def main():
     # Load the definition file
@@ -55,19 +64,28 @@ def main():
         c_program = c.read()
 
     link_libraries = []
-    dependencies_libraries = []
-    # Inject c files into c_program if language is c
     c_source_files = []
+    injected_lua_files = []
+    dependency_libraries = []
+
+    # Inject c files into c_program if language is c
     if(language == 'c'):
-        c_program = inject_c_files(definition, c_program, c_source_files, link_libraries)
+        c_program = inject_c_files(definition, c_program, c_source_files)
 
     # Inject rust files into c_program if language is rust
     if(language == 'rust'):
         c_program = inject_rust_files(definition, c_program)
 
     # Inject lua files into c_program always to load lua files and replace placeholders
     if(language == 'lua'):
-        c_program = inject_lua_files(definition, c_program, link_libraries, dependencies_libraries)
+        c_program = inject_lua_files(definition, c_program, injected_lua_files)
+
+    # Load all libraries
+    c_program = load_libraries(definition, c_program, link_libraries, c_source_files, injected_lua_files,dependency_libraries)
+
+    # Inject the lua files into the c program
+    c_program = c_program.replace(
+        '__INJECT_LUA_FILES__', '\n'.join(injected_lua_files))
 
     #Generate compile target file
     debug_print('Start to generate complie.c')
@@ -119,20 +137,23 @@ def main():
     # Add the compile file and link libraries
     cmd.extend(['/tmp/compile.c', quote('/lua-{}/src/liblua.a'.format(os.environ.get('LUA_VERSION')))])
     cmd.extend([quote(v.filepath) for v in link_libraries])
+    cmd.extend([quote(v.filepath) for v in dependency_libraries])
+    cmd.extend(config.get_extra_args())
 
     # Export the main function and malloc for the runtime
     cmd.extend(['-s', 'EXPORTED_FUNCTIONS=["_malloc", "_main"]'])
 
     # Export the runtime methods cwrap
-    cmd.extend(['-lm', '-ldl', '-o', definition.get_output_file(), '-s', 'EXPORTED_RUNTIME_METHODS=["cwrap", "ccall", "UTF8ToString", "lengthBytesUTF8", "FS"]'])
+    cmd.extend(['-lm', '-ldl', '-o', definition.get_output_file(), '-s', 'EXPORTED_RUNTIME_METHODS=["cwrap"]'])
 
     debug_print('Compile command is {}'.format(' '.join(cmd)))
     shell_exec(*cmd)
 
     # add metering library
     # meter_cmd = ['node', '/opt/node/apply-metering.cjs']
     # shell_exec(*meter_cmd)
-    shell_exec(*['rm', os.path.join(os.getcwd(), 'process.js')])
+    if config.keep_js is False:
+        shell_exec(*['rm', os.path.join(os.getcwd(), 'process.js')])
 
 if __name__ == '__main__':
     main()

dev-cli/container/src/ao_module_lib/config.py

@@ -35,7 +35,7 @@ def mb_to_bytes(megabytes):
     'xxl': {
         'stack_size': mb_to_bytes(96), 
         'initial_memory': mb_to_bytes(128), 
-        'maximum_memory': mb_to_bytes(1024)
+        'maximum_memory': mb_to_bytes(4096)
     },
 }
 
@@ -44,6 +44,8 @@ class Config():
     stack_size = 0
     initial_memory = 0
     maximum_memory = 0
+    extra_compile_args = []
+    keep_js = False
 
     def __init__(self, config_file):
 
@@ -65,9 +67,18 @@ def __init__(self, config_file):
                 self.stack_size = data.get('stack_size', self.stack_size)
                 self.initial_memory = data.get('initial_memory', self.initial_memory)
                 self.maximum_memory = data.get('maximum_memory', self.maximum_memory)
+                self.extra_compile_args = data.get('extra_compile_args', self.extra_compile_args)
+                self.keep_js = data.get('keep_js', self.keep_js)
 
     def setValuesByPreset(self, preset):
         self.preset = preset
         self.stack_size = presets[self.preset]['stack_size']
         self.initial_memory = presets[self.preset]['initial_memory']
-        self.maximum_memory = presets[self.preset]['maximum_memory']
+        self.maximum_memory = presets[self.preset]['maximum_memory']
+
+    def get_extra_args(self):
+        args = []
+        for val in self.extra_compile_args:
+            args.extend(['{}'.format(val)])
+        debug_print('Extra compile args: {}'.format(args))
+        return args

dev-cli/container/src/ao_module_lib/languages/c.py

@@ -6,19 +6,15 @@ def inject_c_files(definition: Definition, c_program: str, c_source_files: list,
 
     c_header_files = []
 
-    c_libraries = glob.glob('/src/**/*.a', recursive=True)
     c_source_files += glob.glob('/src/**/*.c', recursive=True)
     c_source_files += glob.glob('/src/**/*.cpp', recursive=True)
     c_header_files += glob.glob('/src/**/*.h', recursive=True)    
     c_header_files += glob.glob('/src/**/*.hpp', recursive=True)
-
-    for lib in c_libraries:
-        link_libraries.append(BundleFile(lib))
+
 
     c_program = 'const char *process_handle(const char *arg_0, const char *arg_1);\n' + c_program
     c_program = c_program.replace('__FUNCTION_DECLARATIONS__', definition.make_c_function_delarations())
 
     c_program = c_program.replace('__LUA_BASE__', "")
     c_program = c_program.replace('__LUA_MAIN__', "")
-    c_program = c_program.replace('__INJECT_LUA_FILES__', "")
     return c_program

dev-cli/container/src/ao_module_lib/languages/lua.py

@@ -13,25 +13,13 @@
 LUAROCKS_LOCAL_MODULE_DIR = '/src/modules' # os.path.abspath('/src/modules')
 LUAROCKS_LOCAL_MODULE_PREFIX_RE = re.compile(re.escape(LUAROCKS_LOCAL_MODULE_DIR) + '\/share\/lua\/\d+.\d+/')
 
-def __get_uname():
-    uname = ''
-    try:
-        uname, _ = shell_exec('uname', '-s')
-    except (subprocess.CalledProcessError):
-        uname = 'Unknown'
 
-    return uname
 
-
-def inject_lua_files(definition: Definition, c_program: str, link_libraries: list, dependency_libraries: list):
-
-    uname = __get_uname()
+def inject_lua_files(definition: Definition, c_program: str, injected_lua_files: list):
 
     lua_files = []
-    library_files = []
-    dependency_libraries = []
 
-    # entry_file = definition.get_entry_file()
+
     entry_file = '/opt/loader.lua'
 
     if not is_lua_source_file(entry_file):
@@ -44,17 +32,9 @@ def inject_lua_files(definition: Definition, c_program: str, link_libraries: lis
     local_include_prefix_re = re.compile(re.escape(local_include_dir + '/'))
 
     bundle_files = glob.glob('/src/**/*.lua', recursive=True)
-    bundle_files += glob.glob('/src/**/*.so', recursive=True)
-
-    # Optional dependencies
+    bundle_files = list(filter(lambda x: not x.startswith("/src/libs"), bundle_files))
     bundle_files += glob.glob(local_include_dir + '/**/*.lua', recursive=True)
-    bundle_files += glob.glob(local_include_dir + '/**/*.so', recursive=True)
-    bundle_files += glob.glob(LUAROCKS_LOCAL_MODULE_DIR +
-                              '/lib/lua/**/*.so', recursive=True)
-    bundle_files += glob.glob(LUAROCKS_LOCAL_MODULE_DIR +
-                              '/share/lua/**/*.lua', recursive=True)
-    debug_print('Start to factory and distinguish module files')
-    debug_print(NM)
+
     for bundle in bundle_files:
         if is_lua_source_file(bundle):
             debug_print('Lua file found: {}'.format(bundle))
@@ -63,40 +43,9 @@ def inject_lua_files(definition: Definition, c_program: str, link_libraries: lis
             lua_files.append(LuaFile(bundle, basename))
             continue
 
-        if is_binary_library(bundle):
-            try:
-                nm, _ = shell_exec(NM, bundle)
-                is_module = False
-                if re.search(r'T _?luaL_newstate', nm):
-                    if re.search(r'U _?dlopen', nm):
-                        if uname in ['Linux', 'SunOS', 'Darwin']:
-                            libdl_option = '-ldl'
-
-                else:
-                    for luaopen in re.finditer(r'[^dD] _?luaopen_([0-9a-zA-Z!"#\$%&\'\(\)\*\+,\-\.\/:;\<=\>\?@\[\]^_`\{\|\}~]+)', nm):
-                        debug_print('luaopen_{} function found. add to library in {}'.format(
-                            luaopen.group(1), bundle))
-                        library_files.append(
-                            ModuleFile(bundle, luaopen.group(1)))
-                        is_module = True
-
-                if is_module:
-                    link_libraries.append(BundleFile(bundle))
-                else:
-                    dependency_libraries.append(BundleFile(bundle))
-
-            except (subprocess.CalledProcessError):
-                print(NM + ' command failed')
-                return
 
     debug_print('===== Bundle Lua files ======')
     debug_print('\n'.join([v.filepath for v in lua_files]))
-    debug_print('===== Library files =====')
-    debug_print('\n'.join([v.filepath for v in library_files]))
-    debug_print('===== Link libraries =====')
-    debug_print('\n'.join([v.filepath for v in link_libraries]))
-    debug_print('===== Dependency libraries =====')
-    debug_print('\n'.join([v.filepath for v in dependency_libraries]))
 
     with open('/opt/main.lua', mode='r') as lua:
         lua_program = lua.read()
@@ -107,7 +56,6 @@ def inject_lua_files(definition: Definition, c_program: str, link_libraries: lis
         p = main_file.read()
         c_program = c_program.replace('__LUA_MAIN__', encode_hex_literals(p))
 
-    inject_lua_files = []
     for i, f in enumerate(lua_files):
         with open(f.filepath, mode='r') as l:
             lines = l.readlines()
@@ -119,22 +67,13 @@ def inject_lua_files(definition: Definition, c_program: str, link_libraries: lis
                 # shebang
                 lines = lines[1:]
 
-            inject_lua_files.extend([
+            injected_lua_files.extend([
                 '  static const unsigned char lua_require_{}[] = {{{}}};'.format(
                     i, encode_hex_literals('\n'.join(lines))),
                 '  lua_pushlstring(L, (const char*)lua_require_{}, sizeof(lua_require_{}));'.format(i, i),
                 '  lua_setfield(L, -2, "{}");\n'.format(f.module_name)
             ])
 
-    for f in library_files:
-        inject_lua_files.extend([
-            '  int luaopen_{}(lua_State* L);'.format(f.module_name),
-            '  lua_pushcfunction(L, luaopen_{});'.format(f.module_name),
-            '  lua_setfield(L, -2, "{}");'.format(f.basename)
-        ])
     c_program = c_program.replace(
         '__FUNCTION_DECLARATIONS__', definition.make_function_delarations())
-    c_program = c_program.replace(
-        '__INJECT_LUA_FILES__', '\n'.join(inject_lua_files))
-
     return c_program