Implement configurable codegen pipelines

docs/GettingStarted/debug.md

@@ -0,0 +1,35 @@
+## Debugging
+
+VAST makes use of the MLIR infrastructure to facilitate the reproduction of crashes within the pipeline. You can refer to the MLIR documentation on [crash and failure reproduction](https://mlir.llvm.org/docs/PassManagement/#crash-and-failure-reproduction) for more details. We provide a similar set of options in vast-front to aid in debugging.
+
+### Generating Crash Reproducers
+
+To generate a minimal reproducer for a crashed pipeline of `vast-front`, use the following option:
+
+```
+-vast-emit-crash-reproducer="reproducer.mlir" 
+```
+
+This option disables multithreading to ensure a comprehensive crash report. You can then load and examine the crash report using the following command:
+
+```
+vast-opt -run-reproducer reproducer.mlir
+```
+
+### Pipeline Information
+
+To obtain a detailed insight into the pipeline, you can use the following option of `vast-front`:
+
+```
+-vast-print-pipeline
+```
+
+This option dumps the pipeline string to the standard error stream. You can use this information for a more specific investigation of the pipeline. Execute the pipeline with the printed string using the following command:
+
+```
+vast-opt --pass-pipeline="pipeline-string"
+```
+
+### Debug Pipeline
+
+With the `-vast-debug` option, you get more detailed crash reports. It shows MLIR operations when there's an error and provides current stack traces.

include/vast/CodeGen/CodeGen.hpp

@@ -450,7 +450,7 @@ namespace vast::cg
                     VAST_UNIMPLEMENTED;
                 } else if (body) {
                     if (mlir::failed(build_function_body(body))) {
-                        VAST_UNREACHABLE("failed function body codegen");
+                        VAST_FATAL("failed function body codegen");
                     }
                 } else {
                     VAST_UNIMPLEMENTED_MSG("no definition for emitted function");

include/vast/CodeGen/CodeGenContext.hpp

@@ -215,7 +215,7 @@ namespace vast::cg
                 if (const auto *d = llvm::dyn_cast< clang::NamedDecl >(dctx)) {
                     name += get_decl_name(d);
                 } else {
-                    VAST_UNREACHABLE("unknown decl context: {0}", dctx->getDeclKindName());
+                    VAST_FATAL("unknown decl context: {0}", dctx->getDeclKindName());
                 }
 
                 name += "::";

include/vast/CodeGen/CodeGenDeclVisitor.hpp

@@ -185,7 +185,7 @@ namespace vast::cg {
                     return fn;
                 }
 
-                VAST_UNREACHABLE("NYI");
+                VAST_UNIMPLEMENTED;
 
                 // TODO: clang checks here if this is a llvm::GlobalAlias... how will we
                 // support this?
@@ -282,7 +282,7 @@ namespace vast::cg {
                 return fn;
             }
 
-            VAST_UNREACHABLE("codegen of incomplete function");
+            VAST_FATAL("codegen of incomplete function");
         }
 
         vast_function get_addr_of_function(
@@ -486,7 +486,7 @@ namespace vast::cg {
                 case clang::SC_PrivateExtern: return hl::StorageClass::sc_private_extern;
                 case clang::SC_Register: return hl::StorageClass::sc_register;
             }
-            VAST_UNREACHABLE("unknown storage class");
+            VAST_UNIMPLEMENTED_MSG("unknown storage class");
         }
 
         hl::TSClass VisitThreadStorageClass(const clang::VarDecl *decl) const {
@@ -496,7 +496,7 @@ namespace vast::cg {
                 case clang::TSCS_thread_local: return hl::TSClass::tsc_cxx_thread;
                 case clang::TSCS__Thread_local: return hl::TSClass::tsc_c_thread;
             }
-            VAST_UNREACHABLE("unknown storage class");
+            VAST_UNIMPLEMENTED_MSG("unknown thread storage class");
         }
 
         operation VisitVarDecl(const clang::VarDecl *decl) {
@@ -713,7 +713,7 @@ namespace vast::cg {
                 case clang::AccessSpecifier::AS_none:
                     return hl::AccessSpecifier::as_none;
             }
-            VAST_UNREACHABLE("unknown access specifier");
+            VAST_UNIMPLEMENTED_MSG("unknown access specifier");
         }
 
         //

include/vast/CodeGen/CodeGenStmtVisitor.hpp

@@ -747,7 +747,7 @@ namespace vast::cg {
                 return VisitFunctionDeclRefExpr(expr);
             }
 
-            VAST_UNREACHABLE("unknown underlying declaration to be referenced");
+            VAST_UNIMPLEMENTED_MSG("unknown underlying declaration to be referenced");
         }
 
         Operation *VisitPredefinedExpr(const clang::PredefinedExpr *expr)

include/vast/CodeGen/UnreachableVisitor.hpp

@@ -9,32 +9,32 @@ namespace vast::cg
     template< typename derived_t >
     struct unreach_stmt_visitor {
         operation Visit(const clang::Stmt *stmt) {
-            VAST_UNREACHABLE("unsupported stmt: {0}", stmt->getStmtClassName());
+            VAST_FATAL("unsupported stmt: {0}", stmt->getStmtClassName());
         }
     };
 
     template< typename derived_t >
     struct unreach_decl_visitor {
         operation Visit(const clang::Decl *decl) {
-            VAST_UNREACHABLE("unsupported decl: {0}", decl->getDeclKindName());
+            VAST_FATAL("unsupported decl: {0}", decl->getDeclKindName());
         }
     };
 
     template< typename derived_t >
     struct unreach_type_visitor {
         mlir_type Visit(clang::QualType type) {
-            VAST_UNREACHABLE("unsupported type: {0}", type.getAsString());
+            VAST_FATAL("unsupported type: {0}", type.getAsString());
         }
 
         mlir_type Visit(const clang::Type *type) {
-            VAST_UNREACHABLE("unsupported type: {0}", type->getTypeClassName());
+            VAST_FATAL("unsupported type: {0}", type->getTypeClassName());
         }
     };
 
     template< typename derived_t >
     struct unreach_attr_visitor {
         mlir_attr Visit(const clang::Attr *attr) {
-            VAST_UNREACHABLE("unsupported attr: {0}", attr->getSpelling());
+            VAST_FATAL("unsupported attr: {0}", attr->getSpelling());
         }
     };
 

include/vast/Conversion/Passes.hpp

@@ -1,4 +1,4 @@
 // Copyright (c) 2022-present, Trail of Bits, Inc.
 
 #pragma once
 
@@ -20,6 +20,8 @@
 #include <vast/Dialect/Core/CoreDialect.hpp>
 #include <vast/Dialect/ABI/ABIDialect.hpp>
 
+#include <vast/Util/Pipeline.hpp>
+
 #include <memory>
 
 namespace vast
@@ -59,26 +61,16 @@
     #define GEN_PASS_REGISTRATION
     #include "vast/Conversion/Passes.h.inc"
 
-    // TODO(conv): Define dependencies between these.
-    static inline void build_abi_pipeline(mlir::PassManager &pm)
+    namespace conv::pipeline
     {
-        pm.addPass(createEmitABIPass());
-        pm.addPass(createLowerABIPass());
-    }
+        pipeline_step_ptr abi();
+        pipeline_step_ptr irs_to_llvm();
+        pipeline_step_ptr core_to_llvm();
 
-    static inline void build_to_ll_pipeline(mlir::PassManager &pm)
-    {
-        pm.addPass(createHLToLLFuncPass());
-        pm.addPass(createHLToLLVarsPass());
-        pm.addPass(createHLToLLCFPass());
-        pm.addPass(createHLEmitLazyRegionsPass());
-        pm.addPass(createHLToLLGEPsPass());
-    }
-
-    static inline void build_to_llvm_pipeline(mlir::PassManager &pm)
-    {
-        pm.addPass(createIRsToLLVMPass());
-        pm.addPass(createCoreToLLVMPass());
-    }
+        pipeline_step_ptr to_ll();
+
+        pipeline_step_ptr to_llvm();
+
+    } // namespace conv::pipeline
 
 } // namespace vast

include/vast/Conversion/TypeConverters/HLToStd.hpp

@@ -227,7 +227,7 @@ namespace vast::conv::tc {
                     case 128:
                         return mlir::FloatType::getF128(&mctx);
                     default:
-                        VAST_UNREACHABLE("Cannot lower float bitsize {0}", target_bw);
+                        VAST_FATAL("Cannot lower float bitsize {0}", target_bw);
                 }
             };
         }

include/vast/Dialect/HighLevel/HighLevelTypes.hpp

@@ -166,7 +166,7 @@ namespace vast::hl
             .Case< LongDoubleType >([&] (auto t) { return fty::getF80(ctx);  })
             .Case< Float128Type   >([&] (auto t) { return fty::getF128(ctx); })
             .Default([] (auto t) {
-                VAST_UNREACHABLE("unknown float type: {0}", format_type(t));
+                VAST_FATAL("unknown float type: {0}", format_type(t));
                 return mlir_type();
             });
     }

include/vast/Dialect/HighLevel/Passes.hpp

@@ -11,10 +11,11 @@ VAST_RELAX_WARNINGS
 VAST_UNRELAX_WARNINGS
 
 #include <vast/Dialect/HighLevel/HighLevelDialect.hpp>
+#include <vast/Util/Pipeline.hpp>
+
 #include <memory>
 
-namespace vast::hl
-{
+namespace vast::hl {
     std::unique_ptr< mlir::Pass > createHLLowerTypesPass();
 
     std::unique_ptr< mlir::Pass > createExportFnInfoPass();
@@ -25,20 +26,18 @@ namespace vast::hl
 
     std::unique_ptr< mlir::Pass > createSpliceTrailingScopes();
 
-    std::unique_ptr< mlir::Pass > createHLCanonicalizePass();
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "vast/Dialect/HighLevel/Passes.h.inc"
+
+    namespace pipeline {
+        pipeline_step_ptr canonicalize();
 
-    void registerHLToLLVMIR(mlir::DialectRegistry &);
-    void registerHLToLLVMIR(mlir::MLIRContext &);
+        pipeline_step_ptr desugar();
 
-    /// Generate the code for registering passes.
-    #define GEN_PASS_REGISTRATION
-    #include "vast/Dialect/HighLevel/Passes.h.inc"
+        pipeline_step_ptr simplify();
 
-    static inline void build_simplify_hl_pipeline(mlir::PassManager &pm)
-    {
-        pm.addPass(createHLLowerTypesPass());
-        pm.addPass(createDCEPass());
-        pm.addPass(createLowerTypeDefsPass());
-    }
+        pipeline_step_ptr stdtypes();
+    } // namespace pipeline
 
 } // namespace vast::hl

include/vast/Dialect/HighLevel/Passes.td

@@ -90,17 +90,4 @@ def SpliceTrailingScopes : Pass<"vast-hl-splice-trailing-scopes", "mlir::ModuleO
   let constructor = "vast::hl::createSpliceTrailingScopes()";
 }
 
-def HLCanonicalize : Pass<"vast-hl-canonicalize", "mlir::ModuleOp"> {
-  let summary = "Canonicalize hl dialect.";
-  let description = [{
-    This pass inserts returns with void values where missing and removes surplus skips.
-  }];
-
-  let constructor = "vast::hl::createHLCanonicalizePass()";
-  let dependentDialects = [
-    "vast::hl::HighLevelDialect",
-    "vast::core::CoreDialect"
-  ];
-}
-
 #endif // VAST_DIALECT_HIGHLEVEL_PASSES_TD

include/vast/Frontend/Action.hpp

@@ -12,6 +12,7 @@ VAST_UNRELAX_WARNINGS
 #include "vast/Frontend/CompilerInstance.hpp"
 #include "vast/Frontend/FrontendAction.hpp"
 #include "vast/Frontend/Options.hpp"
+#include "vast/Frontend/Targets.hpp"
 
 namespace llvm {
     class LLVMIRContext;

include/vast/Frontend/Consumer.hpp

@@ -1,4 +1,4 @@
 // Copyright (c) 2023-present, Trail of Bits, Inc.
 
 #pragma once
 
@@ -12,6 +12,7 @@
 #include "vast/Frontend/Diagnostics.hpp"
 #include "vast/Frontend/FrontendAction.hpp"
 #include "vast/Frontend/Options.hpp"
+#include "vast/Frontend/Targets.hpp"
 
 #include "vast/CodeGen/CodeGenContext.hpp"
 #include "vast/CodeGen/CodeGenDriver.hpp"
@@ -22,9 +23,6 @@
 
     using clang_ast_consumer = clang::ASTConsumer;
 
-    // TODO: Introduce helper wrapper on top of `vast_args`?
-    enum class target_dialect { high_level, low_level, llvm };
-
     using backend = clang::BackendAction;
 
     struct vast_consumer : clang_ast_consumer
@@ -63,8 +61,6 @@
 
       protected:
 
-        void compile_via_vast(vast_module mod, mcontext_t *mctx);
-
         virtual void anchor() {}
 
         action_options opts;
@@ -102,6 +98,10 @@
             target_dialect target, owning_module_ref mod, mcontext_t *mctx
         );
 
+        void process_mlir_module(
+            target_dialect target, mlir::ModuleOp mod, mcontext_t *mctx
+        );
+
         output_type action;
         output_stream_ptr output_stream;
     };

include/vast/Frontend/FrontendAction.hpp

@@ -16,14 +16,6 @@ namespace vast::cc {
     using frontend_action = clang::ASTFrontendAction;
     using plugin_ast_action = clang::PluginASTAction;
 
-    enum class output_type {
-        emit_assembly,
-        emit_mlir,
-        emit_llvm,
-        emit_obj,
-        none
-    };
-
     static inline action_options options(compiler_instance &ci) {
         return {
             .headers = ci.getHeaderSearchOpts(),

include/vast/Frontend/Options.hpp

@@ -1,4 +1,4 @@
 // Copyright (c) 2022-present, Trail of Bits, Inc.
 
 #pragma once
 
@@ -47,7 +47,6 @@
     struct vast_args
     {
         using option_list = std::vector< string_ref >;
-        using maybe_option_list = std::optional< option_list >;
 
         argv_storage args;
 
@@ -60,7 +59,7 @@
         std::optional< string_ref > get_option(string_ref opt) const;
 
         // from option of form -vast-"name"="value1;value2;value3" returns list of values
-        maybe_option_list get_options_list(string_ref opt) const;
+        std::optional< option_list > get_options_list(string_ref opt) const;
 
         void push_back(arg_t arg);
     };
@@ -71,14 +70,18 @@
         constexpr string_ref emit_llvm = "emit-llvm";
         constexpr string_ref emit_obj  = "emit-obj";
         constexpr string_ref emit_asm  = "emit-asm";
-
         constexpr string_ref emit_mlir = "emit-mlir";
 
-        constexpr string_ref show_locs = "show-locs";
-        constexpr string_ref locs_as_meta_ids = "locs-as-meta-ids";
+        constexpr string_ref print_pipeline = "print-pipeline";
+        constexpr string_ref emit_crash_reproducer = "emit-crash-reproducer";
 
+        constexpr string_ref disable_multithreading = "disable-multithreading";
+        constexpr string_ref debug = "debug";
 
-        constexpr string_ref opt_pipeline  = "pipeline";
+        constexpr string_ref simplify = "simplify";
+
+        constexpr string_ref show_locs = "show-locs";
+        constexpr string_ref locs_as_meta_ids = "locs-as-meta-ids";
 
         constexpr string_ref disable_vast_verifier = "disable-vast-verifier";
         constexpr string_ref vast_verify_diags = "verify-diags";