Enhance wasm with checkpoint and restore support (#2333)

- Add wasm_runtime_checkpoint/wasm_runtime_restore API - Support AOT and Classic Interpreter mode checkpoint and debug through OS signal, tested on windows/mac/linux aarch64/x64 - Static instrument the AOT to have the checkpoint and restore switches - Add sub extra library folder for implementing the ckpt-restore - Include extra dependency of yalantinglib Co-authored-by: Aibo Hu <[email protected]> Co-authored-by: kikispace <[email protected]> Co-authored-by: Brian Zhao <[email protected]> Signed-off-by: victoryang00 <[email protected]>
bytecodealliance · May 2, 2024 · 6cd3e89 · 6cd3e89
1 parent 4dbed20
commit 6cd3e89
Show file tree

Hide file tree

Showing 7 changed files with 397 additions and 2 deletions.
diff --git a/.github/workflows/codeql_buildscript.sh b/.github/workflows/codeql_buildscript.sh
@@ -0,0 +1,277 @@
+#!/usr/bin/env bash
+
+sudo apt update
+
+sudo apt install -y build-essential cmake g++-multilib libgcc-11-dev lib32gcc-11-dev ccache ninja-build ccache
+
+WAMR_DIR=${PWD}
+
+# TODO: use pre-built llvm binary to build wamrc to
+#       avoid static code analysing for llvm
+: '
+# build wamrc
+cd ${WAMR_DIR}/wamr-compiler
+./build_llvm.sh
+rm -fr build && mkdir build && cd build
+cmake ..
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build wamrc!"
+    exit 1;
+fi
+'
+
+# build iwasm with default features enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -fr build && mkdir build && cd build
+cmake ..
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with default features enabled!"
+    exit 1;
+fi
+
+# build iwasm with default features enabled on x86_32
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -fr build && mkdir build && cd build
+cmake .. -DWAMR_BUILD_TARGET=X86_32
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with default features enabled on x86_32!"
+    exit 1;
+fi
+
+# build iwasm with classic interpreter enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_INTERP=0
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with classic interpreter enabled!"
+    exit 1;
+fi
+
+# build iwasm with extra features enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -fr build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug \
+    -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIB_PTHREAD_SEMAPHORE=1 \
+    -DWAMR_BUILD_MULTI_MODULE=1 -DWAMR_BUILD_SIMD=1 \
+    -DWAMR_BUILD_TAIL_CALL=1 -DWAMR_BUILD_REF_TYPES=1 \
+    -DWAMR_BUILD_CUSTOM_NAME_SECTION=1 -DWAMR_BUILD_MEMORY_PROFILING=1 \
+    -DWAMR_BUILD_PERF_PROFILING=1 -DWAMR_BUILD_DUMP_CALL_STACK=1 \
+    -DWAMR_BUILD_LOAD_CUSTOM_SECTION=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build wamrc iwasm with extra features enabled!"
+    exit 1;
+fi
+
+# build iwasm with global heap pool enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -fr build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug \
+    -DWAMR_BUILD_ALLOC_WITH_USER_DATA=1 \
+    -DWAMR_DISABLE_STACK_HW_BOUND_CHECK=1 \
+    -DWAMR_BUILD_GLOBAL_HEAP_POOL=1 \
+    -DWAMR_BUILD_GLOBAL_HEAP_SIZE=131072
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with global heap pool enabled!"
+    exit 1;
+fi
+
+# build iwasm with wasi-threads enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -fr build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LIB_WASI_THREADS=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with wasi-threads enabled!"
+    exit 1;
+fi
+
+# build iwasm with GC enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_GC=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with GC enabled!"
+    exit 1;
+fi
+
+# build iwasm with exception handling enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_EXCE_HANDLING=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with exception handling enabled!"
+    exit 1;
+fi
+
+# build iwasm with memory64 enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MEMORY64=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with memory64 enabled!"
+    exit 1;
+fi
+
+# build iwasm with hardware boundary check disabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_DISABLE_HW_BOUND_CHECK=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with hardware boundary check disabled!"
+    exit 1;
+fi
+
+# build iwasm with quick AOT entry disabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_QUICK_AOT_ENTRY=0
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with quick AOT entry disabled!"
+    exit 1;
+fi
+
+# build iwasm with wakeup of blocking operations disabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_DISABLE_WAKEUP_BLOCKING_OP=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with wakeup of blocking operations disabled!"
+    exit 1;
+fi
+
+# build iwasm with module instance context disabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MODULE_INST_CONTEXT=0 \
+         -DWAMR_BUILD_LIBC_BUILTIN=0 -DWAMR_BUILD_LIBC_WASI=0
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with module instance context disabled!"
+    exit 1;
+fi
+
+# build iwasm with libc-uvwasi enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -fr build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LIBC_UVWASI=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with libc-uvwasi enabled!"
+    exit 1;
+fi
+
+# build iwasm with fast jit lazy mode enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_FAST_JIT_DUMP=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with fast jit lazy mode enabled!"
+    exit 1;
+fi
+
+# build iwasm with fast jit eager mode enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_FAST_JIT_DUMP=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with fast jit eager mode enabled!"
+    exit 1;
+fi
+
+# TODO: use pre-built llvm binary to build llvm-jit and multi-tier-jit
+: '
+# build iwasm with llvm jit lazy mode enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build llvm jit lazy mode enabled!"
+    exit 1;
+fi
+
+# build iwasm with llvm jit eager mode enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build llvm jit eager mode enabled!"
+    exit 1;
+fi
+
+# build iwasm with multi-tier jit enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \
+                                    -DWAMR_BUILD_FAST_JIT_DUMP=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with multi-tier jit enabled!"
+    exit 1;
+fi
+'
+
+# build iwasm with wasm mini-loader enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MINI_LOADER=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build with wasm mini-loader enabled!"
+    exit 1;
+fi
+
+# build iwasm with source debugging enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_DEBUG_INTERP=1 -DWAMR_BUILD_DEBUG_AOT=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with source debugging enabled!"
+    exit 1;
+fi
+
+# build iwasm with AOT static PGO enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_STATIC_PGO=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with AOT static PGO enabled!"
+    exit 1;
+fi
+
+# build iwasm with configurable bounds checks enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_CONFIGUABLE_BOUNDS_CHECKS=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with configurable bounds checks enabled!"
+    exit 1;
+fi
+
+# build iwasm with linux perf support enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux/
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LINUX_PERF=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with linux perf support enabled!"
+    exit 1;
+fi
diff --git a/.github/workflows/codeql_fail_on_error.py b/.github/workflows/codeql_fail_on_error.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+# Return whether SARIF file contains error-level results
+def codeql_sarif_contain_error(filename):
+    with open(filename, 'r') as f:
+        s = json.load(f)
+
+    for run in s.get('runs', []):
+        rules_metadata = run['tool']['driver']['rules']
+        if not rules_metadata:
+            rules_metadata = run['tool']['extensions'][0]['rules']
+
+        for res in run.get('results', []):
+            if 'ruleIndex' in res:
+                rule_index = res['ruleIndex']
+            elif 'rule' in res and 'index' in res['rule']:
+                rule_index = res['rule']['index']
+            else:
+                continue
+            try:
+                rule_level = rules_metadata[rule_index]['defaultConfiguration']['level']
+            except IndexError as e:
+                print(e, rule_index, len(rules_metadata))
+            else:
+                if rule_level == 'error':
+                    return True
+    return False
+
+if __name__ == "__main__":
+    if codeql_sarif_contain_error(sys.argv[1]):
+        sys.exit(1)
diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c
@@ -1420,6 +1420,21 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     /* Get param cell number */
     param_cell_num = func_type->param_cell_num;
 
+    // #if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING !=
+    // 0)
+    //     if (comp_ctx->enable_aux_stack_frame) {
+    //         LLVMValueRef func_idx_const;
+
+    //         if (!(func_idx_const = I32_CONST(func_idx))) {
+    //             aot_set_last_error("llvm build const failed.");
+    //             return false;
+    //         }
+    //         if (!call_aot_alloc_frame_func(comp_ctx, func_ctx,
+    //         func_idx_const))
+    //             return false;
+    //     }
+    // #endif
+
     /* Allocate memory for parameters.
      * Parameters layout:
      *   - exec env

diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h
@@ -460,6 +460,13 @@ WASM_RUNTIME_API_EXTERN wasm_module_t
 wasm_runtime_load_ex(uint8_t *buf, uint32_t size, const LoadArgs *args,
                      char *error_buf, uint32_t error_buf_size);
 
+/**
+ * Load a WASM module with specified load argument.
+ */
+WASM_RUNTIME_API_EXTERN wasm_module_t
+wasm_runtime_load_ex(uint8_t *buf, uint32_t size, const LoadArgs *args,
+                     char *error_buf, uint32_t error_buf_size);
+
 /**
  * Load a WASM module from a specified WASM or AOT section list.
  *
@@ -590,6 +597,16 @@ wasm_runtime_instantiate_ex(const wasm_module_t module,
                             const InstantiationArgs *args, char *error_buf,
                             uint32_t error_buf_size);
 
+/**
+ * Instantiate a WASM module, with specified instantiation arguments
+ *
+ * Same as wasm_runtime_instantiate, but it also allows overwriting maximum memory
+ */
+WASM_RUNTIME_API_EXTERN wasm_module_inst_t
+wasm_runtime_instantiate_ex(const wasm_module_t module,
+                         const InstantiationArgs *args,
+                         char *error_buf, uint32_t error_buf_size);
+
 /**
  * Set the running mode of a WASM module instance, override the
  * default running mode of the runtime. Note that it only makes sense when

diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c
@@ -3363,6 +3363,9 @@ wasm_module_malloc_internal(WASMModuleInstance *module_inst,
     /* TODO: Memory64 size check based on memory idx type */
     bh_assert(size <= UINT32_MAX);
 
+    /* TODO: Memory64 size check based on memory idx type */
+    bh_assert(size <= UINT32_MAX);
+
     if (!memory) {
         wasm_set_exception(module_inst, "uninitialized memory");
         return 0;
@@ -3374,7 +3377,7 @@ wasm_module_malloc_internal(WASMModuleInstance *module_inst,
     else if (module_inst->e->malloc_function && module_inst->e->free_function) {
         if (!execute_malloc_function(
                 module_inst, exec_env, module_inst->e->malloc_function,
-                module_inst->e->retain_function, size, &offset)) {
+                module_inst->e->retain_function, (uint32)size, &offset)) {
             return 0;
         }
         /* If we use app's malloc function,
@@ -3474,7 +3477,7 @@ wasm_module_free_internal(WASMModuleInstance *module_inst,
                  && module_inst->e->free_function && memory->memory_data <= addr
                  && addr < memory_data_end) {
             execute_free_function(module_inst, exec_env,
-                                  module_inst->e->free_function, ptr);
+                                  module_inst->e->free_function, (uint32)ptr);
         }
     }
 }