gufi_vt cleanup

gufi_sqlite3 now takes in 1 required positional argument indicating where gufi_vt is instead of including the source addqueryfuncs.h and histogram.h use SQLITE_EXTENSION_INIT3 when compiling for gufi_vt grouped together internal functions exposed for addhistfuncs right above addhistfuncs added level column to gufi_vt
mar-file-system · Feb 4, 2025 · ffd0e21 · ffd0e21
1 parent 430744b
commit ffd0e21
Show file tree

Hide file tree

Showing 11 changed files with 99 additions and 72 deletions.
diff --git a/contrib/longitudinal_snapshot.py b/contrib/longitudinal_snapshot.py
@@ -695,6 +695,8 @@ def create_longitudinal_snapshot(args):
     cmd = [
         '@CMAKE_BINARY_DIR@/src/gufi_sqlite3',
 
+        '@CMAKE_BINARY_DIR@/src/gufi_vt',
+
         args.outdb,
 
         'ATTACH \'file:{0}?mode=ro\' AS {1};'.format(args.flatdb, FLATDB),

diff --git a/include/addqueryfuncs.h b/include/addqueryfuncs.h
@@ -65,7 +65,12 @@ OF SUCH DAMAGE.
 #ifndef ADDQUERYFUNCS_H
 #define ADDQUERYFUNCS_H
 
+#ifdef SQLITE_CORE
 #include <sqlite3.h>
+#else
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT3
+#endif
 
 #include "bf.h"
 #include "histogram.h"
@@ -74,8 +79,7 @@ OF SUCH DAMAGE.
 extern "C" {
 #endif
 
-/* list of functions to add to a SQLite3 db handle that do not have user data/context */
-
+/* prototypes for addqueryfuncs */
 void uidtouser(sqlite3_context *context, int argc, sqlite3_value **argv);
 void gidtogroup(sqlite3_context *context, int argc, sqlite3_value **argv);
 void modetotxt(sqlite3_context *context, int argc, sqlite3_value **argv);
@@ -89,6 +93,8 @@ void stdevp_final(sqlite3_context *context);
 void median_step(sqlite3_context *context, int argc, sqlite3_value **argv);
 void median_final(sqlite3_context *context);
 
+/* add functions to a SQLite3 db handle that do not have user data/context */
+/* sqlite3_create_function must be called in the same compilation context as gufi_vt */
 static inline int addqueryfuncs(sqlite3 *db) {
     return !(
         (sqlite3_create_function(db,   "uidtouser",           1,   SQLITE_UTF8,
@@ -112,9 +118,10 @@ static inline int addqueryfuncs(sqlite3 *db) {
         (sqlite3_create_function(db,   "median",              1,   SQLITE_UTF8,
                                  NULL, NULL,  median_step,         median_final) == SQLITE_OK) &&
         addhistfuncs(db)
-        );
+    );
 }
 
+/* add functions to a SQLite3 db handle that require gufi_query context */
 int addqueryfuncs_with_context(sqlite3 *db, struct work *work);
 
 #ifdef __cplusplus

diff --git a/include/histogram.h b/include/histogram.h
@@ -68,7 +68,12 @@ OF SUCH DAMAGE.
 #include <stddef.h>
 #include <time.h>
 
+#ifdef SQLITE_CORE
 #include <sqlite3.h>
+#else
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT3
+#endif
 
 #ifdef __cplusplus
 extern "C" {
@@ -78,9 +83,6 @@ extern "C" {
  * Public API for parsing returned strings.
  *
  * These structs are intended for external use.
- *
- * Ignore the *_step and *_final functions. They are sqlite3 UDFs that
- * need to be exposed here to get linking to work for some reason.
  */
 
 /* ********************************************* */
@@ -108,8 +110,6 @@ typedef struct log2_hist {
     size_t ge;       /* len >= 2^count */
 } log2_hist_t;
 
-void log2_hist_step(sqlite3_context *context, int argc, sqlite3_value **argv);
-void log2_hist_final(sqlite3_context *context);
 log2_hist_t *log2_hist_parse(const char *str);
 void log2_hist_free(log2_hist_t *hist);
 /* ********************************************* */
@@ -128,8 +128,6 @@ typedef struct mode_hist {
     size_t buckets[512];
 } mode_hist_t;
 
-void mode_hist_step(sqlite3_context *context, int argc, sqlite3_value **argv);
-void mode_hist_final(sqlite3_context *context);
 mode_hist_t *mode_hist_parse(const char *str);
 void mode_hist_free(mode_hist_t *hist);
 /* ********************************************* */
@@ -169,8 +167,6 @@ typedef struct time_hist {
     time_t ref;
 } time_hist_t;
 
-void time_hist_step(sqlite3_context *context, int argc, sqlite3_value **argv);
-void time_hist_final(sqlite3_context *context);
 time_hist_t *time_hist_parse(const char *str);
 void time_hist_free(time_hist_t *hist);
 /* ********************************************* */
@@ -198,9 +194,6 @@ typedef struct category_hist {
     size_t count;
 } category_hist_t;
 
-void category_hist_step(sqlite3_context *context, int argc, sqlite3_value **argv);
-void category_hist_combine_step(sqlite3_context *context, int argc, sqlite3_value **argv);
-void category_hist_final(sqlite3_context *context);
 category_hist_t *category_hist_parse(const char *str);
 category_hist_t *category_hist_combine(category_hist_t *lhs, category_hist_t *rhs);
 void category_hist_free(category_hist_t *hist);
@@ -220,12 +213,24 @@ typedef struct mode_count {
     size_t count;
 } mode_count_t;
 
-void mode_count_final(sqlite3_context *context);
 mode_count_t *mode_count_parse(const char *str);
 void mode_count_free(mode_count_t *mc);
 /* ********************************************* */
 
+/* prototypes for addhistfuncs */
+void log2_hist_step(sqlite3_context *context, int argc, sqlite3_value **argv);
+void log2_hist_final(sqlite3_context *context);
+void mode_hist_step(sqlite3_context *context, int argc, sqlite3_value **argv);
+void mode_hist_final(sqlite3_context *context);
+void time_hist_step(sqlite3_context *context, int argc, sqlite3_value **argv);
+void time_hist_final(sqlite3_context *context);
+void category_hist_step(sqlite3_context *context, int argc, sqlite3_value **argv);
+void category_hist_combine_step(sqlite3_context *context, int argc, sqlite3_value **argv);
+void category_hist_final(sqlite3_context *context);
+void mode_count_final(sqlite3_context *context);
+
 /* use this to add histogram functions to a sqlite database handle */
+/* sqlite3_create_function must be called in the same compilation context as gufi_vt */
 static inline int addhistfuncs(sqlite3 *db) {
     return (
         (sqlite3_create_function(db,   "log2_hist",             2,  SQLITE_UTF8,

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -124,6 +124,7 @@ set(GUFI_SOURCES
 )
 add_library(GUFI STATIC ${GUFI_SOURCES})
 add_dependencies(GUFI install_dependencies)
+target_compile_definitions(GUFI PUBLIC SQLITE_CORE)
 set_source_files_properties(bf.c PROPERTIES COMPILE_FLAGS -DGUFI_VERSION="${VERSION_STRING}")
 
 #If the GPFS library exists, build the gpfs scan tool
@@ -207,6 +208,7 @@ add_library(gufi_query_lib OBJECT
 )
 
 add_dependencies(gufi_query_lib GUFI)
+target_compile_definitions(gufi_query_lib PUBLIC SQLITE_CORE)
 
 build_and_install_one(${BIN} TRUE gufi_query
   gufi_query/main.c

diff --git a/src/gufi_sqlite3.c b/src/gufi_sqlite3.c
@@ -69,11 +69,8 @@ OF SUCH DAMAGE.
 #include "dbutils.h"
 #include "print.h"
 
-/* don't ask */
-#define SQLITE_CORE
-#include "../src/gufi_vt.c"
-
 static void sub_help(void) {
+    printf("gufi_vt                  location of gufi_vt runtime loadable extension\n");
     printf("db                       db file path\n");
     printf("SQL                      SQL statements to run\n");
     printf("\n");
@@ -84,7 +81,9 @@ static void sub_help(void) {
 
 int main(int argc, char *argv[]) {
     struct input in;
-    process_args_and_maybe_exit("hvd:", 0, "[db [SQL]...]", &in);
+    process_args_and_maybe_exit("hvd:", 1, "gufi_vt [db [SQL]...]", &in);
+
+    const char *gufi_vt = argv[idx++];
 
     const int args_left = argc - idx;
     const char *dbname = (args_left == 0)?SQLITE_MEMORY:argv[idx++];
@@ -97,8 +96,16 @@ int main(int argc, char *argv[]) {
         return EXIT_FAILURE;
     }
 
+    char *err = NULL;
+
     /* this calls addqueryfuncs */
-    sqlite3_gufivt_init(db, NULL, NULL);
+    if (sqlite3_load_extension(db, gufi_vt, "sqlite3_gufivt_init", &err) != SQLITE_OK) {
+        fprintf(stderr, "Error: Could not initialize virtual tables: %s\n", err);
+        sqlite3_free(err);
+        input_fini(&in);
+        closedb(db);
+        return EXIT_FAILURE;
+    }
 
     /* no buffering */
     struct OutputBuffer ob;
@@ -114,7 +121,6 @@ int main(int argc, char *argv[]) {
     };
 
     /* if using in-memory db or no SQL statements following db path, read from stdin */
-    char *err = NULL;
     if (args_left < 2) {
         char *line = NULL;
         size_t len = 0;

diff --git a/src/gufi_vt.c b/src/gufi_vt.c
@@ -62,14 +62,14 @@ OF SUCH DAMAGE.
 
 
 
-#include <sqlite3ext.h>
-SQLITE_EXTENSION_INIT1
-
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT1
+
 #include "addqueryfuncs.h"
 #include "bf.h"
 #include "dbutils.h"
@@ -331,9 +331,9 @@ static int gufi_vtConnect(sqlite3 *db,
 #define GUFI_VT_ARG_COLUMNS          "indexroot TEXT HIDDEN, threads INT64 HIDDEN, " \
                                      "T TEXT HIDDEN, S TEXT HIDDEN, "
 
-#define GUFI_VT_EXTRA_COLUMNS        "path TEXT, epath TEXT, fpath TEXT, rpath TEXT, "
-#define GUFI_VT_EXTRA_COLUMNS_SQL    "path(), epath(), fpath(), path(), "
-#define GUFI_VT_EXTRA_COLUMNS_SQL_VR "path(), epath(), fpath(), rpath(sname, sroll), "
+#define GUFI_VT_EXTRA_COLUMNS        "path TEXT, epath TEXT, fpath TEXT, rpath TEXT, level INT64, "
+#define GUFI_VT_EXTRA_COLUMNS_SQL    "path(), epath(), fpath(), path(), level(), "
+#define GUFI_VT_EXTRA_COLUMNS_SQL_VR "path(), epath(), fpath(), rpath(sname, sroll), level(), "
 
 #define GUFI_VT_ALL_COLUMNS          GUFI_VT_ARG_COLUMNS \
                                      GUFI_VT_EXTRA_COLUMNS

diff --git a/test/regression/gufi_sqlite3.expected b/test/regression/gufi_sqlite3.expected
@@ -1,11 +1,12 @@
 # gufi_sqlite3 help
 $ gufi_sqlite3 -h
-usage: gufi_sqlite3 [options] [db [SQL]...]
+usage: gufi_sqlite3 [options] gufi_vt [db [SQL]...]
 options:
   -h                     help
   -v                     version
   -d <delim>             delimiter (one char)  [use 'x' for 0x1E]
 
+gufi_vt                  location of gufi_vt runtime loadable extension
 db                       db file path
 SQL                      SQL statements to run
 
@@ -14,37 +15,37 @@ Dot commands are not supported
 
 
 # new db file + argv
-$ gufi_sqlite3 "test.db" "CREATE TABLE new_table(i INT);" "INSERT INTO new_table VALUES (0), (1), (2);"
+$ gufi_sqlite3 "gufi_vt" "test.db" "CREATE TABLE new_table(i INT);" "INSERT INTO new_table VALUES (0), (1), (2);"
 
 # existing db file + argv
-$ gufi_sqlite3 "test.db" "SELECT * FROM new_table ORDER BY i DESC;"
+$ gufi_sqlite3 "gufi_vt" "test.db" "SELECT * FROM new_table ORDER BY i DESC;"
 2
 1
 0
 
 $ rm -f "test.db"
 
 # new db file + stdin
-$ (echo "CREATE TABLE new_table(i INT);"; echo "INSERT INTO new_table VALUES (3), (4), (5);") | gufi_sqlite3 "test.db"
+$ (echo "CREATE TABLE new_table(i INT);"; echo "INSERT INTO new_table VALUES (3), (4), (5);") | gufi_sqlite3 "gufi_vt" "test.db"
 
 # existing db file + stdin
-$ echo "SELECT * FROM new_table ORDER BY i DESC;" | gufi_sqlite3 "test.db"
+$ echo "SELECT * FROM new_table ORDER BY i DESC;" | gufi_sqlite3 "gufi_vt" "test.db"
 5
 4
 3
 
 # in-memory db
-$ (echo "CREATE TABLE new_table(i INT);"; echo "INSERT INTO new_table VALUES (6), (7), (8);"; echo "SELECT * FROM new_table ORDER BY i DESC;") | gufi_sqlite3
+$ (echo "CREATE TABLE new_table(i INT);"; echo "INSERT INTO new_table VALUES (6), (7), (8);"; echo "SELECT * FROM new_table ORDER BY i DESC;") | gufi_sqlite3 "gufi_vt"
 8
 7
 6
 
 # directory as db
-$ gufi_sqlite3 "search"
+$ gufi_sqlite3 "gufi_vt" "search"
 Error: Could not open database file "search"
 
 # virtual table
-$ echo "SELECT size, path || '/' || name FROM gufi_vt_pentries('prefix') ORDER BY size ASC;" | gufi_sqlite3 -d "|"
+$ echo "SELECT size, path || '/' || name FROM gufi_vt_pentries('prefix') ORDER BY size ASC;" | gufi_sqlite3 -d "|" "gufi_vt"
 0|prefix/old_file
 1|prefix/directory/executable
 2|prefix/directory/readonly
@@ -61,6 +62,6 @@ $ echo "SELECT size, path || '/' || name FROM gufi_vt_pentries('prefix') ORDER B
 1048576|prefix/1MB
 
 # bad SQL
-$ (echo "CREATE TABLE;") | gufi_sqlite3
+$ (echo "CREATE TABLE;") | gufi_sqlite3 "gufi_vt"
 Error: SQL error: near ";": syntax error
 
diff --git a/test/regression/gufi_sqlite3.sh.in b/test/regression/gufi_sqlite3.sh.in
@@ -86,40 +86,40 @@ echo "# ${GUFI_SQLITE3} help" | replace
 run_no_sort "${GUFI_SQLITE3} -h"
 
 echo "# new db file + argv"
-run_no_sort "${GUFI_SQLITE3} \"${DBNAME}\"" \
+run_no_sort "${GUFI_SQLITE3} \"${GUFI_VT}\" \"${DBNAME}\"" \
             "\"CREATE TABLE new_table(i INT);\"" \
             "\"INSERT INTO new_table VALUES (0), (1), (2);\""
 
 echo "# existing db file + argv"
-run_no_sort "${GUFI_SQLITE3} \"${DBNAME}\"" \
+run_no_sort "${GUFI_SQLITE3} \"${GUFI_VT}\" \"${DBNAME}\"" \
             "\"SELECT * FROM new_table ORDER BY i DESC;\""
 
 run_no_sort "rm -f \"${DBNAME}\""
 
 echo "# new db file + stdin"
 run_no_sort "(echo \"CREATE TABLE new_table(i INT);\";" \
             "echo \"INSERT INTO new_table VALUES (3), (4), (5);\") |" \
-            "${GUFI_SQLITE3} \"${DBNAME}\""
+            "${GUFI_SQLITE3} \"${GUFI_VT}\" \"${DBNAME}\""
 
 echo "# existing db file + stdin"
 run_no_sort "echo \"SELECT * FROM new_table ORDER BY i DESC;\" |" \
-            "${GUFI_SQLITE3} \"${DBNAME}\""
+            "${GUFI_SQLITE3} \"${GUFI_VT}\" \"${DBNAME}\""
 
 echo "# in-memory db"
 run_no_sort "(echo \"CREATE TABLE new_table(i INT);\";" \
             "echo \"INSERT INTO new_table VALUES (6), (7), (8);\";" \
             "echo \"SELECT * FROM new_table ORDER BY i DESC;\") |" \
-            "${GUFI_SQLITE3}"
+            "${GUFI_SQLITE3} \"${GUFI_VT}\""
 
 echo "# directory as db"
-run_no_sort "${GUFI_SQLITE3} \"${SEARCH}\""
+run_no_sort "${GUFI_SQLITE3} \"${GUFI_VT}\" \"${SEARCH}\""
 
 echo "# virtual table"
-PATH="@CMAKE_BINARY_DIR@/src:${PATH}" run_no_sort "echo \"SELECT size, path || '/' || name FROM gufi_vt_pentries('${INDEXROOT}') ORDER BY size ASC;\" | ${GUFI_SQLITE3} -d \"|\""
+PATH="@CMAKE_BINARY_DIR@/src:${PATH}" run_no_sort "echo \"SELECT size, path || '/' || name FROM gufi_vt_pentries('${INDEXROOT}') ORDER BY size ASC;\" | ${GUFI_SQLITE3} -d \"|\" \"${GUFI_VT}\""
 
 echo "# bad SQL"
 run_no_sort "(echo \"CREATE TABLE;\") |" \
-            "${GUFI_SQLITE3}"
+            "${GUFI_SQLITE3} \"${GUFI_VT}\""
 ) |& tee "${OUTPUT}"
 
 @DIFF@ @CMAKE_CURRENT_BINARY_DIR@/gufi_sqlite3.expected "${OUTPUT}"