From 911589e3c7cc89ff4a5dc784597fd94124deacc4 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Fri, 17 Jan 2025 13:43:52 -0700 Subject: [PATCH] write to stdout instead of db file column values are prefixed with type and length when -u is passed in gufi_query and gufi_sqlite3 length is 4 digit human readable to avoid confusion between 0xa and newline always pulling path, epath, fpath, and rpath/path into virtual table --- .github/workflows/codecov.yml | 2 +- contrib/treediff.c | 14 +- include/bf.h | 18 + include/dbutils.h | 8 +- .../{validate_inputs.h => handle_sql.h} | 6 +- include/gufi_query/query.h | 2 +- include/print.h | 1 + src/CMakeLists.txt | 2 +- src/bf.c | 10 + src/dbutils.c | 86 +++ src/gufi_query/aggregate.c | 14 +- .../{validate_inputs.c => handle_sql.c} | 45 +- src/gufi_query/main.c | 6 +- src/gufi_query/process_queries.c | 4 +- src/gufi_query/processdir.c | 4 +- src/gufi_query/query.c | 16 +- src/gufi_sqlite3.c | 2 +- src/gufi_vt.c | 500 +++++++++--------- src/print.c | 28 + test/regression/CMakeLists.txt | 2 +- test/regression/gufi_query.expected | 13 + test/regression/gufi_query.sh.in | 6 + test/regression/gufi_vt.expected | 50 +- test/regression/gufi_vt.sh.in | 27 +- test/unit/googletest/PoolArgs.cpp.in | 1 + test/unit/googletest/dbutils.cpp.in | 78 +++ test/unit/googletest/print.cpp | 104 +++- 27 files changed, 698 insertions(+), 351 deletions(-) rename include/gufi_query/{validate_inputs.h => handle_sql.h} (96%) rename src/gufi_query/{validate_inputs.c => handle_sql.c} (79%) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index d0cb5edbb..4aa1830cd 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -102,7 +102,7 @@ jobs: run: ctest || true - name: Delete files not reported for coverage - run: find -name "*.gc*" -a \( -name "gendir.*" -o -name "make_testindex.*" -o -name "bfwreaddirplus2db.*" -o -name "bffuse.*" -o -name "bfresultfuse.*" -o -name "dfw.*" -o -name "tsmepoch2time.*" -o -name "tsmtime2epoch.*" -o -path "*/test/*" \) -delete + run: find -name "*.gc*" -a \( -name "gendir.*" -o -name "make_testindex.*" -o -name "bfwreaddirplus2db.*" -o -name "bffuse.*" -o -name "bfresultfuse.*" -o -name "dfw.*" -o -name "tsmepoch2time.*" -o -name "tsmtime2epoch.*" -o -path "*/test/*" -o -name "gufi_vt.dir" \) -delete - name: Generate Python Coverage Report run: | diff --git a/contrib/treediff.c b/contrib/treediff.c index dc7acbb83..d31a82f35 100644 --- a/contrib/treediff.c +++ b/contrib/treediff.c @@ -255,12 +255,14 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) { /* ********************************************** */ const size_t next_level = cp->level + 1; - struct PrintArgs print; - print.output_buffer = &pa->obufs.buffers[id]; - print.delim = '/'; - print.mutex = pa->obufs.mutex; - print.outfile = stdout; - print.rows = 0; + struct PrintArgs print = { + .output_buffer = &pa->obufs.buffers[id], + .delim = '/', + .mutex = pa->obufs.mutex, + .outfile = stdout, + .rows = 0, + .types = NULL, + }; char *buf[] = {NULL, NULL}; /* passed to print_parallel */ diff --git a/include/bf.h b/include/bf.h index 8ba0272f9..594d9362e 100644 --- a/include/bf.h +++ b/include/bf.h @@ -178,6 +178,24 @@ struct input { refstr_t fin; } sql; + /* + * if outputting to STDOUT or OUTFILE, get list of + * types of final output to prefix columns with + * + * set up by gufi_query but cleaned up by input_fini + */ + struct { + int prefix; + + /* set if not aggregating */ + int *tsum; + int *sum; + int *ent; + + /* set if aggregating */ + int *agg; + } types; + int printdir; int printing; int printheader; diff --git a/include/dbutils.h b/include/dbutils.h index 46678f55a..34ca0ce3b 100644 --- a/include/dbutils.h +++ b/include/dbutils.h @@ -102,14 +102,14 @@ extern const char READDIRPLUS_INSERT[]; /* prefer pentries over entries */ #define ENTRIES "entries" #define ENTRIES_SCHEMA(name, extra_cols) \ - "CREATE TABLE " name "(name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, crtime INT64, ossint1 INT64, ossint2 INT64, ossint3 INT64, ossint4 INT64, osstext1 TEXT, osstext2 TEXT" extra_cols ");" + "CREATE TABLE " name "(" extra_cols "name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, crtime INT64, ossint1 INT64, ossint2 INT64, ossint3 INT64, ossint4 INT64, osstext1 TEXT, osstext2 TEXT);" extern const char ENTRIES_CREATE[]; extern const char ENTRIES_INSERT[]; /* directory metadata + aggregate data */ #define SUMMARY "summary" #define SUMMARY_SCHEMA(name, extra_cols) \ - "CREATE TABLE " name "(name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, rectype INT64, pinode TEXT, isroot INT64, rollupscore INT64" extra_cols ");" + "CREATE TABLE " name "(" extra_cols "name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, rectype INT64, pinode TEXT, isroot INT64, rollupscore INT64);" extern const char SUMMARY_CREATE[]; /* view of summary table with rollups */ @@ -136,7 +136,7 @@ extern const char VRPENTRIES_CREATE[]; /* aggregate data of tree starting at current directory */ #define TREESUMMARY "treesummary" #define TREESUMMARY_SCHEMA(name, extra_cols) \ - "CREATE TABLE " name "(inode TEXT, pinode TEXT, totsubdirs INT64, maxsubdirfiles INT64, maxsubdirlinks INT64, maxsubdirsize INT64, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, totextdbs INT64, rectype INT64, uid INT64, gid INT64" extra_cols ");" + "CREATE TABLE " name "(" extra_cols "inode TEXT, pinode TEXT, totsubdirs INT64, maxsubdirfiles INT64, maxsubdirlinks INT64, maxsubdirsize INT64, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, totextdbs INT64, rectype INT64, uid INT64, gid INT64);" #define TREESUMMARY_CREATE \ DROP_TABLE(TREESUMMARY) \ TREESUMMARY_SCHEMA(TREESUMMARY, "") @@ -268,6 +268,8 @@ enum CheckRollupScore { int bottomup_collect_treesummary(sqlite3 *db, const char *dirname, sll_t *subdirs, const enum CheckRollupScore check_rollupscore); +int *get_col_types(sqlite3 *db, const refstr_t *sql, int *cols); + #ifdef __cplusplus } #endif diff --git a/include/gufi_query/validate_inputs.h b/include/gufi_query/handle_sql.h similarity index 96% rename from include/gufi_query/validate_inputs.h rename to include/gufi_query/handle_sql.h index e5781aecc..f29e2cc3d 100644 --- a/include/gufi_query/validate_inputs.h +++ b/include/gufi_query/handle_sql.h @@ -62,11 +62,11 @@ OF SUCH DAMAGE. -#ifndef GUFI_QUERY_VALIDATE_INPUTS_H -#define GUFI_QUERY_VALIDATE_INPUTS_H +#ifndef GUFI_QUERY_HANDLE_SQL_H +#define GUFI_QUERY_HANDLE_SQL_H #include "bf.h" -int validate_inputs(struct input *in); +int handle_sql(struct input *in); #endif diff --git a/include/gufi_query/query.h b/include/gufi_query/query.h index cd8afe743..308e30be8 100644 --- a/include/gufi_query/query.h +++ b/include/gufi_query/query.h @@ -71,7 +71,7 @@ OF SUCH DAMAGE. void querydb(struct work *work, const char *dbname, const size_t dbname_len, - sqlite3 *db, const char *query, + sqlite3 *db, const char *query, int *types, PoolArgs_t *pa, int id, int (*callback)(void *, int, char **, char**), int *rc); diff --git a/include/print.h b/include/print.h index 3542cfa83..dbb340a1b 100644 --- a/include/print.h +++ b/include/print.h @@ -82,6 +82,7 @@ typedef struct PrintArgs { pthread_mutex_t *mutex; /* mutex for printing to stdout */ FILE *outfile; size_t rows; /* number of rows returned by the query */ + int *types; /* if set, prefix output with 1 octet type and 4 digit human readable length */ /* size_t printed; /\* number of records printed by the callback *\/ */ } PrintArgs_t; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5612f0e95..e3d847b3b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -200,10 +200,10 @@ add_library(gufi_query_lib OBJECT gufi_query/aggregate.c gufi_query/external.c gufi_query/gqw.c + gufi_query/handle_sql.c gufi_query/process_queries.c gufi_query/processdir.c gufi_query/query.c - gufi_query/validate_inputs.c ) add_dependencies(gufi_query_lib GUFI) diff --git a/src/bf.c b/src/bf.c index bc8844ca4..c0297c2f1 100644 --- a/src/bf.c +++ b/src/bf.c @@ -124,6 +124,10 @@ struct input *input_init(struct input *in) { void input_fini(struct input *in) { if (in) { + free(in->types.agg); + free(in->types.ent); + free(in->types.sum); + free(in->types.tsum); sll_destroy(&in->external_attach, free); trie_free(in->skip); } @@ -156,6 +160,7 @@ void print_help(const char* prog_name, case 'd': printf(" -d delimiter (one char) [use 'x' for 0x%02X]", (uint8_t)fielddelim); break; case 'o': printf(" -o output file (one-per-thread, with thread-id suffix)"); break; case 'O': printf(" -O output DB"); break; + case 'u': printf(" -u prefix output with 1 octet type and 4 digit human readable length"); break; /* need to use text to avoid \x0a confusion */ case 'I': printf(" -I SQL init"); break; case 'T': printf(" -T SQL for tree-summary table"); break; case 'S': printf(" -S SQL for summary table"); break; @@ -208,6 +213,7 @@ void show_input(struct input* in, int retval) { printf("in.maxthreads = %zu\n", in->maxthreads); printf("in.delim = '%c'\n", in->delim); printf("in.andor = %d\n", (int) in->andor); + printf("in.types.prefix = %d\n", in->types.prefix); printf("in.process_xattrs = %d\n", in->process_xattrs); printf("in.nobody.uid = %" STAT_uid "\n", in->nobody.uid); printf("in.nobody.gid = %" STAT_gid "\n", in->nobody.gid); @@ -353,6 +359,10 @@ int parse_cmd_line(int argc, INSTALL_STR(&in->outname, optarg); break; + case 'u': + in->types.prefix = 1; + break; + case 'I': // SQL initializations INSTALL_STR(&in->sql.init, optarg); break; diff --git a/src/dbutils.c b/src/dbutils.c index 634f33df8..434ccde8d 100644 --- a/src/dbutils.c +++ b/src/dbutils.c @@ -76,6 +76,7 @@ OF SUCH DAMAGE. #include "dbutils.h" #include "external.h" #include "histogram.h" +#include "trie.h" static const char SQLITE_MEMORY_ARRAY[] = ":memory:"; const char *SQLITE_MEMORY = SQLITE_MEMORY_ARRAY; @@ -1190,3 +1191,88 @@ int bottomup_collect_treesummary(sqlite3 *db, const char *dirname, sll_t *subdir return inserttreesumdb(dirname, db, &tsum, 0, 0, 0); } + +/* + * subset of known string to SQLite type conversions + * + * https://www.sqlite.org/datatype3.html + * https://www.sqlite.org/c3ref/c_blob.html + */ +static trie_t *sqlite3_types(void) { + trie_t *types = trie_alloc(); + + trie_insert(types, "INT", 3, (void *) (uintptr_t) SQLITE_INTEGER, NULL); + trie_insert(types, "INTEGER", 7, (void *) (uintptr_t) SQLITE_INTEGER, NULL); + trie_insert(types, "INT64", 5, (void *) (uintptr_t) SQLITE_INTEGER, NULL); + + trie_insert(types, "FLOAT", 5, (void *) (uintptr_t) SQLITE_FLOAT, NULL); + trie_insert(types, "DOUBLE", 6, (void *) (uintptr_t) SQLITE_FLOAT, NULL); + trie_insert(types, "REAL", 4, (void *) (uintptr_t) SQLITE_FLOAT, NULL); + + trie_insert(types, "TEXT", 4, (void *) (uintptr_t) SQLITE_TEXT, NULL); + + trie_insert(types, "BLOB", 4, (void *) (uintptr_t) SQLITE_BLOB, NULL); + + trie_insert(types, "NULL", 4, (void *) (uintptr_t) SQLITE_NULL, NULL); + + return types; +} + +int *get_col_types(sqlite3 *db, const refstr_t *sql, int *cols) { + int rc = SQLITE_OK; + + /* parse sql */ + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, sql->data, sql->len, &stmt, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error: Could not prepare '%s' for getting column types: %s (%d)\n", + sql->data, sqlite3_errstr(rc), rc); + return NULL; + } + + /* /\* */ + /* * need to step if calling sqlite3_column_type, but requires */ + /* * that the table has at least 1 row of actual values */ + /* *\/ */ + /* rc = sqlite3_step(stmt); */ + /* if (rc != SQLITE_ROW) { */ + /* fprintf(stderr, "Error: Failed to evaluate SQL statement '%s': %s (%d)\n", */ + /* sql->data, sqlite3_errstr(rc), rc); */ + /* return NULL; */ + /* } */ + + /* get column count */ + *cols = sqlite3_column_count(stmt); + if (*cols == 0) { + fprintf(stderr, "Error: '%s' was detected to have 0 columns\n", sql->data); + sqlite3_finalize(stmt); + return NULL; + } + + trie_t *str2type = sqlite3_types(); + + /* get each column's type */ + int *types = malloc(*cols * sizeof(int)); + for(int i = 0; i < *cols; i++) { + const char *type = sqlite3_column_decltype(stmt, i); + if (!type) { + types[i] = SQLITE_NULL; + continue; + } + + const size_t type_len = strlen(type); + + void *sql_type = NULL; + if (trie_search(str2type, type, type_len, &sql_type) == 1) { + types[i] = (uintptr_t) sql_type; + } + else { + types[i] = 0; /* unknown type */ + } + } + + trie_free(str2type); + + sqlite3_finalize(stmt); + return types; +} diff --git a/src/gufi_query/aggregate.c b/src/gufi_query/aggregate.c index c5531d69c..fe6593b1d 100644 --- a/src/gufi_query/aggregate.c +++ b/src/gufi_query/aggregate.c @@ -157,12 +157,14 @@ int aggregate_process(Aggregate_t *aggregate, struct input *in) { /* normally expect STDOUT/OUTFILE to have SQL to run, but OUTDB can have SQL to run as well */ if ((in->output != OUTDB) || in->sql.agg.len) { - PrintArgs_t pa; - pa.output_buffer = &aggregate->ob; - pa.delim = in->delim; - pa.mutex = NULL; - pa.outfile = aggregate->outfile; - pa.rows = 0; + PrintArgs_t pa = { + .output_buffer = &aggregate->ob, + .delim = in->delim, + .mutex = NULL, + .outfile = aggregate->outfile, + .rows = 0, + .types = in->types.agg, + }; char *err = NULL; if (sqlite3_exec(aggregate->db, in->sql.agg.data, print_parallel, &pa, &err) != SQLITE_OK) { diff --git a/src/gufi_query/validate_inputs.c b/src/gufi_query/handle_sql.c similarity index 79% rename from src/gufi_query/validate_inputs.c rename to src/gufi_query/handle_sql.c index 7d64831dd..c97a5b827 100644 --- a/src/gufi_query/validate_inputs.c +++ b/src/gufi_query/handle_sql.c @@ -63,10 +63,13 @@ OF SUCH DAMAGE. #include +#include -#include "gufi_query/validate_inputs.h" +#include "dbutils.h" +#include "gufi_query/handle_sql.h" +#include "template_db.h" -int validate_inputs(struct input *in) { +int handle_sql(struct input *in) { /* * - Leaves are final outputs * - OUTFILE/OUTDB + aggregation will create per thread and final aggregation files @@ -140,5 +143,43 @@ int validate_inputs(struct input *in) { } } + /* now that the SQL has been validated, generate types if necessary */ + if ((in->types.prefix == 1) && ((in->output == STDOUT) || (in->output == OUTFILE))) { + /* have to create temporary db since there is no guarantee of a db yet */ + sqlite3 *db = opendb(SQLITE_MEMORY, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + 0, 0, create_dbdb_tables, NULL); + if (!db) { + return -1; + } + + int cols = 0; /* discarded */ + + /* if not aggregating, get types for T, S, and E */ + if (!in->sql.init_agg.len) { + if (in->sql.tsum.len) { + in->types.tsum = get_col_types(db, &in->sql.tsum, &cols); + } + if (in->sql.sum.len) { + in->types.sum = get_col_types(db, &in->sql.sum, &cols); + } + if (in->sql.ent.len) { + in->types.ent = get_col_types(db, &in->sql.ent, &cols); + } + } + /* types for G */ + else { + char *err = NULL; + if (sqlite3_exec(db, in->sql.init_agg.data, NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "Error: Init failed while getting column types: %s\n", err); + sqlite3_free(err); + closedb(db); + return -1; + } + in->types.agg = get_col_types(db, &in->sql.agg, &cols); + } + + closedb(db); + } + return 0; } diff --git a/src/gufi_query/main.c b/src/gufi_query/main.c index 4f79eb6dc..398e8cb85 100644 --- a/src/gufi_query/main.c +++ b/src/gufi_query/main.c @@ -78,8 +78,8 @@ OF SUCH DAMAGE. #include "gufi_query/aggregate.h" #include "gufi_query/gqw.h" +#include "gufi_query/handle_sql.h" #include "gufi_query/processdir.h" -#include "gufi_query/validate_inputs.h" static void sub_help(void) { printf("GUFI_index find GUFI index here\n"); @@ -93,9 +93,9 @@ int main(int argc, char *argv[]) /* Callers provide the options-string for get_opt(), which will */ /* control which options are parsed for each program. */ struct input in; - process_args_and_maybe_exit("hHvT:S:E:an:jo:d:O:I:F:y:z:J:K:G:mB:wxk:M:s:" COMPRESS_OPT "Q:", 1, "GUFI_index ...", &in); + process_args_and_maybe_exit("hHvT:S:E:an:jo:d:O:uI:F:y:z:J:K:G:mB:wxk:M:s:" COMPRESS_OPT "Q:", 1, "GUFI_index ...", &in); - if (validate_inputs(&in) != 0) { + if (handle_sql(&in) != 0) { input_fini(&in); return EXIT_FAILURE; } diff --git a/src/gufi_query/process_queries.c b/src/gufi_query/process_queries.c index cd11d5dfd..9ed62a8ee 100644 --- a/src/gufi_query/process_queries.c +++ b/src/gufi_query/process_queries.c @@ -255,7 +255,7 @@ int process_queries(PoolArgs_t *pa, if (in->sql.sum.len) { recs=1; /* set this to one record - if the sql succeeds it will set to 0 or 1 */ /* put in the path relative to the user's input */ - querydb(&gqw->work, dbname, dbname_len, db, in->sql.sum.data, pa, id, print_parallel, &recs); + querydb(&gqw->work, dbname, dbname_len, db, in->sql.sum.data, in->types.sum, pa, id, print_parallel, &recs); } else { recs = 1; } @@ -265,7 +265,7 @@ int process_queries(PoolArgs_t *pa, /* if we have recs (or are running an OR) query the entries table */ if (recs > 0) { if (in->sql.ent.len) { - querydb(&gqw->work, dbname, dbname_len, db, in->sql.ent.data, pa, id, print_parallel, &recs); /* recs is not used */ + querydb(&gqw->work, dbname, dbname_len, db, in->sql.ent.data, in->types.ent, pa, id, print_parallel, &recs); /* recs is not used */ } } } diff --git a/src/gufi_query/processdir.c b/src/gufi_query/processdir.c index 36a88cea8..eb9531640 100644 --- a/src/gufi_query/processdir.c +++ b/src/gufi_query/processdir.c @@ -214,14 +214,14 @@ int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) { if (in->andor == AND) { /* make sure the treesummary table exists */ querydb(&gqw->work, dbname, dbname_len, db, "SELECT name FROM " ATTACH_NAME ".sqlite_master " - "WHERE (type == 'table') AND (name == '" TREESUMMARY "');", + "WHERE (type == 'table') AND (name == '" TREESUMMARY "');", NULL, pa, id, count_rows, &recs); if (recs < 1) { recs = -1; } else { /* run in->sql.tsum */ - querydb(&gqw->work, dbname, dbname_len, db, in->sql.tsum.data, pa, id, print_parallel, &recs); + querydb(&gqw->work, dbname, dbname_len, db, in->sql.tsum.data, in->types.tsum, pa, id, print_parallel, &recs); } } /* this is an OR or we got a record back. go on to summary/entries */ diff --git a/src/gufi_query/query.c b/src/gufi_query/query.c index c7970c97c..209cd9ab3 100644 --- a/src/gufi_query/query.c +++ b/src/gufi_query/query.c @@ -69,16 +69,18 @@ OF SUCH DAMAGE. /* wrapper wround sqlite3_exec to pass arguments and check for errors */ void querydb(struct work *work, const char *dbname, const size_t dbname_len, - sqlite3 *db, const char *query, + sqlite3 *db, const char *query, int *types, PoolArgs_t *pa, int id, int (*callback)(void *, int, char **, char**), int *rc) { ThreadArgs_t *ta = &pa->ta[id]; - PrintArgs_t args; - args.output_buffer = &ta->output_buffer; - args.delim = pa->in->delim; - args.mutex = pa->stdout_mutex; - args.outfile = ta->outfile; - args.rows = 0; + PrintArgs_t args = { + .output_buffer = &ta->output_buffer, + .delim = pa->in->delim, + .mutex = pa->stdout_mutex, + .outfile = ta->outfile, + .rows = 0, + .types = types, + }; char *err = NULL; #ifdef SQL_EXEC diff --git a/src/gufi_sqlite3.c b/src/gufi_sqlite3.c index 4578ab8aa..d4277cc08 100644 --- a/src/gufi_sqlite3.c +++ b/src/gufi_sqlite3.c @@ -95,7 +95,6 @@ int main(int argc, char *argv[]) { } addqueryfuncs(db); - addhistfuncs(db); /* no buffering */ struct OutputBuffer ob; @@ -107,6 +106,7 @@ int main(int argc, char *argv[]) { .mutex = NULL, .outfile = stdout, .rows = 0, + .types = NULL, }; char *err = NULL; diff --git a/src/gufi_vt.c b/src/gufi_vt.c index d7e1c0c1b..f617a16b3 100644 --- a/src/gufi_vt.c +++ b/src/gufi_vt.c @@ -104,20 +104,39 @@ SQLITE_EXTENSION_INIT1 * arguments are all optional, but are positional, and appear in the * following order: * - # of threads - * - output directory (where to temporarily put aggregated data: default: pwd) - * - I * - T * - S - * - E - * - K - * - J - * - G - * - F * * The schemas of all 6 of the corresponding tables and views are * recreated here, and thus all columns are accessible. */ +typedef struct gufi_query_sql { + const char *I; + const char *T; + const char *S; + const char *E; + const char *K; + const char *J; + const char *G; + const char *F; +} gq_sql_t; + +typedef struct gufi_vtab { + sqlite3_vtab base; + gq_sql_t sql; /* not const to allow for T and S to be modified */ +} gufi_vtab; + +typedef struct gufi_vtab_cursor { + sqlite3_vtab_cursor base; + + FILE *output; /* result of popen */ + char *row; /* current row */ + ssize_t len; /* length of current row */ + + sqlite_int64 rowid; /* current row id */ +} gufi_vtab_cursor; + /* Calling addqueryfuncs causes SQLite3 to set up incorrectly, leading to a segfault at load time */ static int addvtfuncs(sqlite3 *db) { return !( @@ -144,16 +163,7 @@ static int addvtfuncs(sqlite3 *db) { ); } -typedef struct gufi_query_sql { - const char *I; - const char *T; - const char *S; - const char *E; - const char *K; - const char *J; - const char *G; - const char *F; -} gq_sql_t; +#define delim "\x1E" /* record separator */ /* * run gufi_query, aggregating results into a single db file @@ -162,33 +172,16 @@ typedef struct gufi_query_sql { * everything to link dynamically */ static int gufi_query_aggregate_db(const char *indexroot, const char *threads, const gq_sql_t *sql, - const char *prefix, char **outdb, char **errmsg) { - /* allow for aggregate databases to be placed anywhere */ - const size_t outdb_len = (prefix?strlen(prefix):0) + 15; - *outdb = sqlite3_malloc(outdb_len + 1); - if (prefix) { - snprintf(*outdb, outdb_len + 1, "%s/gufi_vt.XXXXXX", prefix); - } - else { - snprintf(*outdb, outdb_len + 1, "gufi_vt.XXXXXX"); - } - - const int fd = mkstemp(*outdb); - if (fd < 0) { - const int err = errno; - *errmsg = sqlite3_mprintf("mkstemp('%s') failed: %s (%d)", *outdb, strerror(err), err); - goto error; - } - close(fd); - - const char *argv[23] = { + FILE **output, char **errmsg) { + const char *argv[24] = { "gufi_query", - "-O", *outdb, + "-u", + "-d", delim, }; #define set_argv(argc, argv, flag, value) if (value) { argv[argc++] = flag; argv[argc++] = value; } - int argc = 3; + int argc = 4; set_argv(argc, argv, "-n", threads); set_argv(argc, argv, "-I", sql->I); set_argv(argc, argv, "-T", sql->T); @@ -202,68 +195,35 @@ static int gufi_query_aggregate_db(const char *indexroot, const char *threads, c argv[argc++] = indexroot; argv[argc] = NULL; - const pid_t pid = fork(); - if (pid == -1) { - const int err = errno; - *errmsg = sqlite3_mprintf("Failed to fork: %s (%d)", strerror(err), err); - goto error; + size_t len = 0; + for(int i = 0; i < argc; i++) { + len += strlen(argv[i]) + 3; /* + 2 for quotes around each argument + 1 for space between args */ } - /* child */ - if (pid == 0) { - /* using execvp to allow for gufi_query executable to be selected by PATH */ - const int rc = execvp("gufi_query", (char * const *) argv); - if (rc != 0) { - const int err = errno; - fprintf(stderr, "Failed to start gufi_query: %s (%d)\n", strerror(err), err); - sqlite3_free(*outdb); - *outdb = NULL; - exit(EXIT_FAILURE); /* child needs to exit here and now */ - } - /* can't get here */ + /* convert array of args to single string */ + char *cmd = malloc(len + 1); + char *curr = cmd; + for(int i = 0; i < argc; i++) { + /* FIXME: this should use single quotes to avoid potentially processing variables, but needs to be double quotes to handle strings in SQLite properly */ + curr += snprintf(curr, len + 1 - (curr - cmd), "\"%s\" ", argv[i]); } - /* parent */ - int status = EXIT_SUCCESS; - const int rc = waitpid(pid, &status, 0); - if (rc != pid) { + /* pass command to popen */ + FILE *out = popen(cmd, "re"); + + free(cmd); + + if (!out) { const int err = errno; - *errmsg = sqlite3_mprintf("Failed to wait for gufi_query: %s (%d)\n", strerror(err), err); - goto error; + *errmsg = sqlite3_mprintf("popen failed: %s (%d)", strerror(err), err); + return SQLITE_ERROR; } - if (status != EXIT_SUCCESS) { - *errmsg = sqlite3_mprintf("gufi_query returned error: %d\n", status); - goto error; - } + *output = out; return SQLITE_OK; - - error: - remove(*outdb); /* not checking for error */ - sqlite3_free(*outdb); - *outdb = NULL; - return SQLITE_ERROR; } -typedef struct gufi_vtab { - sqlite3_vtab base; - gq_sql_t sql; /* not const to allow for changes */ - - /* TODO: track multiple aggregate result files to not rerun queries unnecessarily */ - /* hash(sql) -> file name */ - char *dbname; /* keep track of aggregate db file name to delete upon unload; */ - /* this is always freed */ -} gufi_vtab; - -typedef struct gufi_vtab_cursor { - sqlite3_vtab_cursor base; - sqlite3 *db; /* the aggregated results database file */ - sqlite3_stmt *stmt; /* compiled SQL pulling from aggregate database */ - sqlite_int64 rowid; /* current row id */ - int res; /* previous sqlite3_step return code */ -} gufi_vtab_cursor; - /* generic connect function */ static int gufi_vtConnect(sqlite3 *db, void *pAux, @@ -281,7 +241,7 @@ static int gufi_vtConnect(sqlite3 *db, pNew = (gufi_vtab *)sqlite3_malloc( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); - pNew->sql = *sql; /* copy to allow for setting SQL of instance without modifying source */ + pNew->sql = *sql; /* sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); */ } @@ -290,41 +250,53 @@ static int gufi_vtConnect(sqlite3 *db, } /* positional arguments to virtual table/table-valued function */ -#define GUFI_VT_COLUMN_INDEXROOT 0 -#define GUFI_VT_COLUMN_THREADS 1 -#define GUFI_VT_COLUMN_OUTPUT_PREFIX 2 -#define GUFI_VT_COLUMN_I 3 -#define GUFI_VT_COLUMN_T 4 -#define GUFI_VT_COLUMN_S 5 -#define GUFI_VT_COLUMN_E 6 -#define GUFI_VT_COLUMN_K 7 -#define GUFI_VT_COLUMN_J 8 -#define GUFI_VT_COLUMN_G 9 -#define GUFI_VT_COLUMN_F 10 - -#define GUFI_VT_HIDDEN_COLUMNS ", indexroot TEXT HIDDEN, threads INT64 HIDDEN, output_prefix TEXT HIDDEN" \ - ", I TEXT HIDDEN" \ - ", T TEXT HIDDEN, S TEXT HIDDEN, E TEXT HIDDEN" \ - ", K TEXT HIDDEN, J TEXT HIDDEN, G TEXT HIDDEN" \ - ", F TEXT HIDDEN" - -#define PENTRIES_SCHEMA(name, extra) \ - "CREATE TABLE " name "(name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, crtime INT64, ossint1 INT64, ossint2 INT64, ossint3 INT64, ossint4 INT64, osstext1 TEXT, osstext2 TEXT, pinode TEXT, ppinode TEXT" extra ");" - -#define VRSUMMARY_SCHEMA(name, extra) \ - "CREATE TABLE " name "(dname TEXT, sname TEXT, sroll INT64, srollsubdirs INT64, name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, rectype INT64, pinode TEXT, isroot INT64, rollupscore INT64" extra ");" - -#define VRPENTRIES_SCHEMA(name, extra) \ - "CREATE TABLE " name "(dname TEXT, sname TEXT, dmode INT64, dnlink INT64, duid INT64, dgid INT64, dsize INT64, dblksize INT64, dblocks INT64, datime INT64, dmtime INT64, dctime INT64, dlinkname TEXT, dtotfile INT64, dtotlinks INT64, dminuid INT64, dmaxuid INT64, dmingid INT64, dmaxgid INT64, dminsize INT64, dmaxsize INT64, dtotzero INT64, dtotltk INT64, dtotmtk INT64, totltm INT64, dtotmtm INT64, dtotmtg INT64, dtotmtt INT64, dtotsize INT64, dminctime INT64, dmaxctime INT64, dminmtime INT64, dmaxmtime INT64, dminatime INT64, dmaxatime INT64, dminblocks INT64, dmaxblocks INT64, dtotxattr INT64, ddepth INT64, dmincrtime INT64, dmaxcrtime INT64, sroll INT64, atroot INT64, srollsubdirs INT64, name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, crtime INT64, ossint1 INT64, ossint2 INT64, ossint3 INT64, ossint4 INT64, osstext1 TEXT, osstext2 TEXT, pinode TEXT, ppinode TEXT" extra ");" +#define GUFI_VT_ARGS_INDEXROOT 0 +#define GUFI_VT_ARGS_THREADS 1 +#define GUFI_VT_ARGS_T 2 +#define GUFI_VT_ARGS_S 3 +#define GUFI_VT_ARGS_COUNT 4 + +#define GUFI_VT_ARG_COLUMNS "indexroot TEXT HIDDEN, threads INT64 HIDDEN, " \ + "T TEXT HIDDEN, S TEXT HIDDEN, " + +#define GUFI_VT_EXTRA_COLUMNS "path TEXT, epath TEXT, fpath TEXT, rpath TEXT, " +#define GUFI_VT_EXTRA_COLUMNS_SQL "path(), epath(), fpath(), path(), " +#define GUFI_VT_EXTRA_COLUMNS_SQL_VR "path(), epath(), fpath(), rpath(sname, sroll), " + +#define GUFI_VT_ALL_COLUMNS GUFI_VT_ARG_COLUMNS \ + GUFI_VT_EXTRA_COLUMNS + +#define PENTRIES_SCHEMA(name, extra_cols) \ + "CREATE TABLE " name "(" extra_cols "name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, crtime INT64, ossint1 INT64, ossint2 INT64, ossint3 INT64, ossint4 INT64, osstext1 TEXT, osstext2 TEXT, pinode TEXT, ppinode TEXT);" + +#define VRSUMMARY_SCHEMA(name, extra_cols) \ + "CREATE TABLE " name "(" extra_cols "dname TEXT, sname TEXT, sroll INT64, srollsubdirs INT64, name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, rectype INT64, pinode TEXT, isroot INT64, rollupscore INT64);" + +#define VRPENTRIES_SCHEMA(name, extra_cols) \ + "CREATE TABLE " name "(" extra_cols "dname TEXT, sname TEXT, dmode INT64, dnlink INT64, duid INT64, dgid INT64, dsize INT64, dblksize INT64, dblocks INT64, datime INT64, dmtime INT64, dctime INT64, dlinkname TEXT, dtotfile INT64, dtotlinks INT64, dminuid INT64, dmaxuid INT64, dmingid INT64, dmaxgid INT64, dminsize INT64, dmaxsize INT64, dtotzero INT64, dtotltk INT64, dtotmtk INT64, totltm INT64, dtotmtm INT64, dtotmtg INT64, dtotmtt INT64, dtotsize INT64, dminctime INT64, dmaxctime INT64, dminmtime INT64, dmaxmtime INT64, dminatime INT64, dmaxatime INT64, dminblocks INT64, dmaxblocks INT64, dtotxattr INT64, ddepth INT64, dmincrtime INT64, dmaxcrtime INT64, sroll INT64, atroot INT64, srollsubdirs INT64, name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, crtime INT64, ossint1 INT64, ossint2 INT64, ossint3 INT64, ossint4 INT64, osstext1 TEXT, osstext2 TEXT, pinode TEXT, ppinode TEXT);" #define INTERMEDIATE "intermediate" #define AGGREGATE "aggregate" -#define SELECT_FROM(name, extra) ("INSERT INTO " INTERMEDIATE " SELECT * FROM " name ";" extra) -#define INSERT_AGG ("INSERT INTO " AGGREGATE " SELECT * FROM " INTERMEDIATE ";") +#define SELECT_FROM(name, extra_sql) \ + (const char *) ( \ + "INSERT INTO " INTERMEDIATE " " \ + "SELECT " GUFI_VT_EXTRA_COLUMNS_SQL "* " \ + "FROM " name ";" extra_sql \ + ) + +#define SELECT_FROM_VR(name, extra_sql) \ + (const char *) ( \ + "INSERT INTO " INTERMEDIATE " " \ + "SELECT " GUFI_VT_EXTRA_COLUMNS_SQL_VR "* " \ + "FROM " name ";" extra_sql \ + ) + +#define INSERT_AGG ("INSERT INTO " AGGREGATE " SELECT * FROM " INTERMEDIATE ";") +#define SELECT_AGG ("SELECT * FROM " AGGREGATE ";") /* generate xConnect function for each virtual table */ -#define gufi_vt_XConnect(name, abbrev, t, s, e) \ +#define gufi_vt_xConnect(name, abbrev, t, s, e, vr) \ static int gufi_vt_ ##abbrev ##Connect(sqlite3 *db, \ void *pAux, \ int argc, const char * const *argv, \ @@ -332,17 +304,17 @@ static int gufi_vtConnect(sqlite3 *db, char **pzErr) { \ /* this is what the virtual table looks like */ \ static const char schema[] = \ - name ##_SCHEMA(name, GUFI_VT_HIDDEN_COLUMNS); \ + name ##_SCHEMA(name, GUFI_VT_ALL_COLUMNS); \ \ /* this is the actual query */ \ static const gq_sql_t sql = { \ - .I = name ##_SCHEMA(INTERMEDIATE, ""), \ - .T = t?SELECT_FROM(name, "SELECT NULL;"):NULL, \ - .S = s?SELECT_FROM(name, ""):NULL, \ - .E = e?SELECT_FROM(name, ""):NULL, \ - .K = name ##_SCHEMA(AGGREGATE, ""), \ + .I = name ##_SCHEMA(INTERMEDIATE, GUFI_VT_EXTRA_COLUMNS), \ + .T = t?(SELECT_FROM(name, "SELECT NULL;")):NULL, \ + .S = s?(vr?SELECT_FROM_VR(name, ""):SELECT_FROM(name, "")):NULL, \ + .E = e?(vr?SELECT_FROM_VR(name, ""):SELECT_FROM(name, "")):NULL, \ + .K = name ##_SCHEMA(AGGREGATE, GUFI_VT_EXTRA_COLUMNS), \ .J = INSERT_AGG, \ - .G = NULL, \ + .G = SELECT_AGG, \ .F = NULL, \ }; \ \ @@ -350,25 +322,26 @@ static int gufi_vtConnect(sqlite3 *db, schema, &sql); \ } -/* generate xConnect for each table/view */ -gufi_vt_XConnect(TREESUMMARY, T, 1, 0, 0) -gufi_vt_XConnect(SUMMARY, S, 0, 1, 0) -gufi_vt_XConnect(ENTRIES, E, 0, 0, 1) -gufi_vt_XConnect(PENTRIES, P, 0, 0, 1) -gufi_vt_XConnect(VRSUMMARY, VRS, 0, 1, 0) -gufi_vt_XConnect(VRPENTRIES, VRP, 0, 0, 1) +/* generate xConnect for each table/view */ +gufi_vt_xConnect(TREESUMMARY, T, 1, 0, 0, 0) +gufi_vt_xConnect(SUMMARY, S, 0, 1, 0, 0) +gufi_vt_xConnect(ENTRIES, E, 0, 0, 1, 0) +gufi_vt_xConnect(PENTRIES, P, 0, 0, 1, 0) +gufi_vt_xConnect(VRSUMMARY, VRS, 0, 1, 0, 1) +gufi_vt_xConnect(VRPENTRIES, VRP, 0, 0, 1, 1) + +/* FIXME: This is probably not correct */ static int gufi_vtBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo) { + /* gufi_vtab *vtab = (gufi_vtab *) tab; */ (void) tab; int argc = 0; /* number of input arguments */ - int ne = 0; /* number of non-EQ constraints */ const struct sqlite3_index_constraint *constraint = pIdxInfo->aConstraint; for(int i = 0; i < pIdxInfo->nConstraint; i++, constraint++) { if (constraint->op != SQLITE_INDEX_CONSTRAINT_EQ) { - ne++; continue; } @@ -376,10 +349,11 @@ static int gufi_vtBestIndex(sqlite3_vtab *tab, continue; } - pIdxInfo->aConstraintUsage[i].argvIndex = i + 1 - ne; /* FIXME: This is probably not correct */ - pIdxInfo->aConstraintUsage[i].omit = 1; - - argc++; + if (constraint->iColumn < GUFI_VT_ARGS_COUNT) { + pIdxInfo->aConstraintUsage[i].argvIndex = constraint->iColumn + 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + argc++; + } } /* index root not found */ @@ -391,9 +365,6 @@ static int gufi_vtBestIndex(sqlite3_vtab *tab, } static int gufi_vtDisconnect(sqlite3_vtab *pVtab) { - gufi_vtab *vtab = (gufi_vtab *) pVtab; - remove(vtab->dbname); /* check for error? */ - sqlite3_free(vtab->dbname); sqlite3_free(pVtab); return SQLITE_OK; } @@ -409,12 +380,8 @@ static int gufi_vtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor) { static int gufi_vtClose(sqlite3_vtab_cursor *cur) { gufi_vtab_cursor *pCur = (gufi_vtab_cursor *) cur; - if (pCur->stmt) { - sqlite3_finalize(pCur->stmt); - } - if (pCur->db) { - sqlite3_close(pCur->db); - } + free(pCur->row); + pCur->row = NULL; sqlite3_free(cur); return SQLITE_OK; } @@ -429,144 +396,193 @@ static int gufi_vtFilter(sqlite3_vtab_cursor *cur, gufi_vtab *vtab = (gufi_vtab *) cur->pVtab; /* indexroot must be present */ - const char *indexroot = (const char *) sqlite3_value_text(argv[GUFI_VT_COLUMN_INDEXROOT]); + const char *indexroot = (const char *) sqlite3_value_text(argv[GUFI_VT_ARGS_INDEXROOT]); const char *threads = NULL; - const char *output_prefix = NULL; - if (argc > GUFI_VT_COLUMN_THREADS) { + if (argc > GUFI_VT_ARGS_THREADS) { /* passing NULL in the SQL will result in a NULL pointer */ - if ((threads = (const char *) sqlite3_value_text(argv[GUFI_VT_COLUMN_THREADS]))) { + if ((threads = (const char *) sqlite3_value_text(argv[GUFI_VT_ARGS_THREADS]))) { + size_t nthreads = 0; if ((sscanf(threads, "%zu", &nthreads) != 1) || (nthreads == 0)) { vtab->base.zErrMsg = sqlite3_mprintf("Bad thread count: '%s'", threads); - sqlite3_free(vtab->dbname); - vtab->dbname = NULL; return SQLITE_CONSTRAINT; } } } - #define set_str(argc, argv, idx, dst) \ - if (argc > idx) { \ - if (sqlite3_value_bytes(argv[idx]) != 0) { \ - dst = (const char *) sqlite3_value_text(argv[idx]); \ - } \ + #define set_str(argc, argv, idx, dst) \ + if (argc > idx) { \ + if (sqlite3_value_bytes(argv[idx]) != 0) { \ + dst = (const char *) sqlite3_value_text(argv[idx]); \ + } \ } - set_str(argc, argv, GUFI_VT_COLUMN_OUTPUT_PREFIX, output_prefix); - - /* change default queries if they were provided */ - set_str(argc, argv, GUFI_VT_COLUMN_I, vtab->sql.I); - set_str(argc, argv, GUFI_VT_COLUMN_T, vtab->sql.T); - set_str(argc, argv, GUFI_VT_COLUMN_S, vtab->sql.S); - set_str(argc, argv, GUFI_VT_COLUMN_E, vtab->sql.E); - set_str(argc, argv, GUFI_VT_COLUMN_K, vtab->sql.K); - set_str(argc, argv, GUFI_VT_COLUMN_J, vtab->sql.J); - set_str(argc, argv, GUFI_VT_COLUMN_G, vtab->sql.G); - set_str(argc, argv, GUFI_VT_COLUMN_F, vtab->sql.F); + /* -T and -S can be changed */ + set_str(argc, argv, GUFI_VT_ARGS_T, vtab->sql.T); + set_str(argc, argv, GUFI_VT_ARGS_S, vtab->sql.S); - /* run gufi_query to get aggregate results */ + /* kick off gufi_query */ const int rc = gufi_query_aggregate_db(indexroot, threads, &vtab->sql, - output_prefix, &vtab->dbname, &vtab->base.zErrMsg); - - if (rc != EXIT_SUCCESS) { - /* output file has already been removed */ + &pCur->output, &vtab->base.zErrMsg); + if (rc != SQLITE_OK) { return SQLITE_ERROR; } - /* open the aggregate db file */ - if (sqlite3_open_v2(vtab->dbname, &pCur->db, SQLITE_OPEN_READONLY, GUFI_SQLITE_VFS) != SQLITE_OK) { - sqlite3_free(vtab->base.zErrMsg); - vtab->base.zErrMsg = sqlite3_mprintf("Could not open aggregate db %s", vtab->dbname); - goto error; - } + pCur->rowid = 0; + pCur->row = NULL; - /* set up SQL for retreiving data from results table */ - static const char SELECT_AGG[] = "SELECT * FROM " AGGREGATE ";"; - if (sqlite3_prepare_v2(pCur->db, SELECT_AGG, sizeof(SELECT_AGG), &pCur->stmt, NULL) != SQLITE_OK) { + /* wait for first row */ + size_t len = 0; + pCur->len = getline(&pCur->row, &len, pCur->output); + + if (pCur->len < 0) { /* failed or reached EOF */ + const int err = errno; sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = sqlite3_mprintf("Could not prepare SQL for aggregate db: %s", - sqlite3_errmsg(pCur->db)); - goto error; + cur->pVtab->zErrMsg = sqlite3_mprintf("Could not read first result: %s (%d)", + strerror(err), err); + return SQLITE_ERROR; } - pCur->rowid = 0; - pCur->res = sqlite3_step(pCur->stmt); /* go to first result */ - if ((pCur->res != SQLITE_ROW) && (pCur->res != SQLITE_DONE)) { - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = sqlite3_mprintf("Could not prepare step into aggregate results: %s", - sqlite3_errmsg(pCur->db)); - goto error; + if (pCur->row[pCur->len - 1] == '\n') { + pCur->row[pCur->len - 1] = '\0'; + pCur->len--; } return SQLITE_OK; - - error: - remove(vtab->dbname); - sqlite3_free(vtab->dbname); - vtab->dbname = NULL; - return SQLITE_ERROR; } static int gufi_vtNext(sqlite3_vtab_cursor *cur) { gufi_vtab_cursor *pCur = (gufi_vtab_cursor *) cur; - pCur->res = sqlite3_step(pCur->stmt); - if ((pCur->res != SQLITE_ROW) && (pCur->res != SQLITE_DONE)) { - return SQLITE_ERROR; + + size_t len = 0; + free(pCur->row); + pCur->row = NULL; + pCur->len = getline(&pCur->row, &len, pCur->output); + + /* no more to read */ + if (pCur->len == -1) { + return SQLITE_OK; } - if (pCur->res == SQLITE_ROW) { - pCur->rowid++; + + /* remove trailing newline */ + if (pCur->row[pCur->len - 1] == '\n') { + pCur->len--; } + pCur->rowid++; + return SQLITE_OK; } static int gufi_vtEof(sqlite3_vtab_cursor *cur) { gufi_vtab_cursor *pCur = (gufi_vtab_cursor *) cur; - const int eof = (pCur->res != SQLITE_ROW); + + const int eof = (pCur->len < 1); if (eof) { - sqlite3_reset(pCur->stmt); + pclose(pCur->output); + pCur->output = NULL; } + return eof; } +static int find_col(const char *str, const size_t len, const char sep, const size_t idx, + int *type, const char **col, size_t *col_len) { + (void) sep; + + if (!str) { + goto error; + } + + char *curr = (char *) str; + size_t i = 0; + while ((i <= idx) && ((size_t) (curr - str) < len)) { + /* type */ + *type = *curr; + curr++; + + /* length */ + /* have to use text instead of binary to distinguish 10 from '\x0a' */ + char buf[5] = {0}; + memcpy(buf, curr, 4); + if (sscanf(buf, "%zu", col_len) != 1) { + goto error; + } + curr += 4; + + /* value */ + *col = curr; + curr += *col_len; + + /* delimiter */ + curr++; + i++; + } + + if (i != (idx + 1)) { + goto error; + } + + return 0; + + error: + *type = 0; + *col = NULL; + *col_len = 0; + return 1; +} + static int gufi_vtColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, - int i) { + int N) { gufi_vtab_cursor *pCur = (gufi_vtab_cursor *) cur; - const int coltype = sqlite3_column_type(pCur->stmt, i); - switch (coltype) { - case SQLITE_INTEGER: - { - const int col = sqlite3_column_int(pCur->stmt, i); - sqlite3_result_int(ctx, col); - } - break; - case SQLITE_FLOAT: - { - const double col = sqlite3_column_int(pCur->stmt, i); - sqlite3_result_double(ctx, col); - } - break; - case SQLITE_TEXT: - { - const char *col = (char *) sqlite3_column_text(pCur->stmt, i); - const int bytes = sqlite3_column_bytes(pCur->stmt, i); - sqlite3_result_text(ctx, col, bytes, SQLITE_TRANSIENT); - } - break; - case SQLITE_BLOB: - { - const void *col = sqlite3_column_blob(pCur->stmt, i); - const int bytes = sqlite3_column_bytes(pCur->stmt, i); - sqlite3_result_blob(ctx, col, bytes, SQLITE_TRANSIENT); - } - break; - case SQLITE_NULL: - default: - sqlite3_result_null(ctx); - break; + int type = 0; + const char *col = NULL; + size_t len = 0; + + if (find_col(pCur->row, pCur->len, delim[0], N - GUFI_VT_ARGS_COUNT, &type, &col, &len) == 0) { + switch(type) { + case SQLITE_INTEGER: + { + int value = 0; + if (sscanf(col, "%d", &value) != 1) { + const int err = errno; + gufi_vtab *vtab = (gufi_vtab *) &pCur->base.pVtab; + vtab->base.zErrMsg = sqlite3_mprintf("Could not parse '%.*s' as a double: %s (%d)\n", + len, col, strerror(err), err); + return SQLITE_ERROR; + } + sqlite3_result_int(ctx, value); + break; + } + case SQLITE_FLOAT: + { + double value = 0; + if (sscanf(col, "%lf", &value) != 1) { + const int err = errno; + gufi_vtab *vtab = (gufi_vtab *) &pCur->base.pVtab; + vtab->base.zErrMsg = sqlite3_mprintf("Could not parse '%.*s' as a double: %s (%d)\n", + len, col, strerror(err), err); + return SQLITE_ERROR; + } + sqlite3_result_double(ctx, value); + break; + } + case SQLITE_TEXT: + case SQLITE_BLOB: + sqlite3_result_text(ctx, col, len, SQLITE_TRANSIENT); + break; + case SQLITE_NULL: + default: + sqlite3_result_text(ctx, col, len, SQLITE_TRANSIENT); + /* sqlite3_result_null(ctx); */ + break; + } + } + else { + sqlite3_result_null(ctx); } return SQLITE_OK; @@ -597,7 +613,7 @@ static const sqlite3_module gufi_vtModule = { 0, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ - 0, /* xFindMethod */ + 0, /* xFindFunction */ 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ diff --git a/src/print.c b/src/print.c index 68e2fde8a..451e676c4 100644 --- a/src/print.c +++ b/src/print.c @@ -72,6 +72,7 @@ int print_parallel(void *args, int count, char **data, char **columns) { PrintArgs_t *print = (PrintArgs_t *) args; struct OutputBuffer *ob = print->output_buffer; + const int *types = print->types; size_t *lens = malloc(count * sizeof(size_t)); size_t row_len = count - 1 + 1; /* one delimiter per column except last column + newline */ @@ -83,6 +84,10 @@ int print_parallel(void *args, int count, char **data, char **columns) { } } + if (types) { + row_len += count * 5; /* 1 octet type and 4 digit human readable length per column */ + } + /* if a row cannot fit the buffer for whatever reason, flush the existing buffer */ if ((ob->capacity - ob->filled) < row_len) { if (print->mutex) { @@ -102,12 +107,28 @@ int print_parallel(void *args, int count, char **data, char **columns) { } const int last = count - 1; for(int i = 0; i < last; i++) { + if (types) { + const char col_type = types[i]; + fwrite(&col_type, sizeof(char), sizeof(col_type), print->outfile); + + char buf[5]; + const size_t len = snprintf(buf, sizeof(buf), "%04zu", lens[i]); + fwrite(buf, sizeof(char), len, print->outfile); + } if (data[i]) { fwrite(data[i], sizeof(char), lens[i], print->outfile); } fwrite(&print->delim, sizeof(char), 1, print->outfile); } /* print last column with no follow up delimiter */ + if (types) { + const char col_type = types[last]; + fwrite(&col_type, sizeof(char), sizeof(col_type), print->outfile); + + char buf[5]; + const size_t len = snprintf(buf, sizeof(buf), "%04zu", lens[last]); + fwrite(buf, sizeof(char), len, print->outfile); + } fwrite(data[last], sizeof(char), lens[last], print->outfile); fwrite("\n", sizeof(char), 1, print->outfile); ob->count++; @@ -122,6 +143,13 @@ int print_parallel(void *args, int count, char **data, char **columns) { char *buf = ob->buf; size_t filled = ob->filled; for(int i = 0; i < count; i++) { + if (types) { + buf[filled] = types[i]; + filled++; + + const ssize_t len = snprintf(&buf[filled], ob->capacity - filled, "%04zu", lens[i]); + filled += len; + } if (data[i]) { memcpy(&buf[filled], data[i], lens[i]); filled += lens[i]; diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index 37eef0379..ea40a9f79 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -84,8 +84,8 @@ set(BINARIES gufi_treesummary_all gufi_query gufi_stat_bin - gufi_vt querydbs + gufi_vt ) set(PYTHON diff --git a/test/regression/gufi_query.expected b/test/regression/gufi_query.expected index b638e2e2c..31db9fd2a 100644 --- a/test/regression/gufi_query.expected +++ b/test/regression/gufi_query.expected @@ -14,6 +14,7 @@ options: -o output file (one-per-thread, with thread-id suffix) -d delimiter (one char) [use 'x' for 0x1E] -O output DB + -u prefix output with 1 octet type and 4 digit human readable length -I SQL init -F SQL cleanup -y minimum level to go down @@ -219,6 +220,18 @@ prefix/repeat_name prefix/unusual#? directory , prefix/unusual#? directory ,/unusual, name?# +# Output TLV columns (no aggregation) +$ gufi_query -u -d " " -n 2 -E "SELECT name, size FROM vrpentries;" "prefix" | sort | head -n 1 | od -x --endian=big +0000000 0330 3030 3331 4b42 2001 3030 3034 3130 +0000020 3234 0a00 +0000023 + +# Output TLV columns (with aggregation) +$ gufi_query -u -d " " -n 2 -a -I "CREATE TABLE out(path TEXT, size INT64);" -K "CREATE TABLE aggregate(path TEXT, size INT64);" -S "INSERT INTO out SELECT rpath(sname, sroll), size FROM vrsummary;" -E "INSERT INTO out SELECT rpath(sname, sroll) || '/' || name, size FROM vrpentries;" -J "INSERT INTO aggregate SELECT path, size FROM out;" -G "SELECT path, size FROM aggregate ORDER BY path ASC LIMIT 1;" "prefix" | od -x --endian=big +0000000 0330 3031 3373 6561 7263 682f 7072 6566 +0000020 6978 2001 3030 3032 3137 0a00 +0000033 + ##################################### # Invalid Inputs # ##################################### diff --git a/test/regression/gufi_query.sh.in b/test/regression/gufi_query.sh.in index 52b2cd448..c22660f6a 100755 --- a/test/regression/gufi_query.sh.in +++ b/test/regression/gufi_query.sh.in @@ -105,6 +105,12 @@ run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -S \"SELECT rpath(sname, sroll) F echo "# Get all directory and non-directory names and their xattrs" run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -S \"SELECT rpath(sname, sroll), xattr_name, xattr_value FROM vrxsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name, xattr_name, xattr_value FROM vrxpentries;\" -x \"${INDEXROOT}\"" +echo "# Output TLV columns (no aggregation)" +run_no_sort "${GUFI_QUERY} -u -d \" \" -n ${THREADS} -E \"SELECT name, size FROM vrpentries;\" \"${INDEXROOT}\" | sort | head -n 1 | od -x --endian=big" + +echo "# Output TLV columns (with aggregation)" +run_no_sort "${GUFI_QUERY} -u -d \" \" -n ${THREADS} -a -I \"CREATE TABLE out(path TEXT, size INT64);\" -K \"CREATE TABLE aggregate(path TEXT, size INT64);\" -S \"INSERT INTO out SELECT rpath(sname, sroll), size FROM vrsummary;\" -E \"INSERT INTO out SELECT rpath(sname, sroll) || '/' || name, size FROM vrpentries;\" -J \"INSERT INTO aggregate SELECT path, size FROM out;\" -G \"SELECT path, size FROM aggregate ORDER BY path ASC LIMIT 1;\" \"${INDEXROOT}\" | od -x --endian=big" + echo "#####################################" echo "# Invalid Inputs #" echo "#####################################" diff --git a/test/regression/gufi_vt.expected b/test/regression/gufi_vt.expected index 9d22cfb45..10a6e3132 100644 --- a/test/regression/gufi_vt.expected +++ b/test/regression/gufi_vt.expected @@ -6,12 +6,6 @@ $ ( echo ".load gufi_vt" echo "SELECT minsize, maxsize, minmtime, maxmtime FROM gufi_vt_treesummary('prefix', 2) ORDER BY minsize ASC, maxsize ASC;" ) | sqlite3 - - - - - - -1|0|-1|0 0|1048576|0|1048576 1|5|1|5 @@ -131,9 +125,8 @@ unusual, name?#|15 # Query entries in directory where name == 'directory' $ ( echo ".load gufi_vt" - echo "SELECT name FROM gufi_vt_pentries('prefix', 2, NULL, NULL, NULL, 'SELECT NULL FROM summary WHERE name == ''directory'';') ORDER BY name ASC, size ASC;" + echo "SELECT name FROM gufi_vt_pentries('prefix', 2, NULL, 'SELECT NULL FROM summary WHERE name == ''directory'';') ORDER BY name ASC, size ASC;" ) | sqlite3 - executable readonly writable @@ -141,9 +134,8 @@ writable # Query directories that contain entries larger than 1024 (only 1: prefix) $ ( echo ".load gufi_vt" - echo "SELECT name, size, mtime FROM gufi_vt_pentries('prefix', 2, NULL, NULL, NULL, 'SELECT NULL FROM summary WHERE maxsize > 1024;') ORDER BY name ASC, size ASC;" + echo "SELECT name, size, mtime FROM gufi_vt_pentries('prefix', 2, NULL, 'SELECT NULL FROM summary WHERE maxsize > 1024;') ORDER BY name ASC, size ASC;" ) | sqlite3 - .hidden|10|10 1KB|1024|1024 1MB|1048576|1048576 @@ -151,6 +143,13 @@ file_symlink|9|9 old_file|0|0 repeat_name|14|14 +# Paths +$ ( + echo ".load gufi_vt" + echo "SELECT path, epath, fpath, rpath FROM gufi_vt_pentries('prefix', 2) WHERE name == '.hidden';" +) | sqlite3 +prefix|prefix|prefix|prefix + # Missing thread count (not an error) $ ( echo ".load gufi_vt" @@ -191,37 +190,6 @@ repeat_name unusual, name?# writable -# Place results in a specific directory -$ ( - echo ".load gufi_vt" - echo "SELECT name, size, mtime FROM gufi_vt_pentries('prefix', 2, 'alt_dir') ORDER BY name ASC, size ASC;" -) | sqlite3 -.hidden|10|10 -1KB|1024|1024 -1MB|1048576|1048576 -directory_symlink|4|4 -executable|1|1 -file_symlink|9|9 -leaf_file1|11|11 -leaf_file2|12|12 -old_file|0|0 -readonly|2|2 -repeat_name|5|5 -repeat_name|14|14 -unusual, name?#|15|15 -writable|3|3 - -# Make sure all types work -$ ( - echo ".load gufi_vt" - echo "SELECT rowid, 1, 1.0, 'text', CAST('blob' AS BLOB), NULL FROM gufi_vt_pentries('prefix', 2) LIMIT 5;" -) | sqlite3 -0|1|1.0|text|blob| -1|1|1.0|text|blob| -2|1|1.0|text|blob| -3|1|1.0|text|blob| -4|1|1.0|text|blob| - # Missing indexroot (error) $ ( echo ".load gufi_vt" diff --git a/test/regression/gufi_vt.sh.in b/test/regression/gufi_vt.sh.in index 9b933ba1c..69d38d18d 100755 --- a/test/regression/gufi_vt.sh.in +++ b/test/regression/gufi_vt.sh.in @@ -67,20 +67,6 @@ source @CMAKE_CURRENT_BINARY_DIR@/setup.sh 1 OUTPUT="gufi_vt.out" LOAD=".load @CMAKE_BINARY_DIR@/src/gufi_vt" -ALT_DIR="$(mktemp -d XXXXXX)" - -cleanup() { - rm -rf "${ALT_DIR}" -} - -cleanup_exit() { - cleanup - setup_cleanup -} - -trap cleanup_exit EXIT - -# no initial cleanup query_vt() { sql="$1" @@ -115,10 +101,13 @@ echo "# Query with WHERE size > 10" query_vt "SELECT name, size FROM gufi_vt_pentries('${INDEXROOT}', ${THREADS}) WHERE size > 10 ORDER BY name ASC, size ASC;" echo "# Query entries in directory where name == 'directory'" -query_vt "SELECT name FROM gufi_vt_pentries('${INDEXROOT}', ${THREADS}, NULL, NULL, NULL, 'SELECT NULL FROM summary WHERE name == ''directory'';') ORDER BY name ASC, size ASC;" +query_vt "SELECT name FROM gufi_vt_pentries('${INDEXROOT}', ${THREADS}, NULL, 'SELECT NULL FROM summary WHERE name == ''directory'';') ORDER BY name ASC, size ASC;" echo "# Query directories that contain entries larger than 1024 (only 1: ${INDEXROOT})" -query_vt "SELECT name, size, mtime FROM gufi_vt_pentries('${INDEXROOT}', ${THREADS}, NULL, NULL, NULL, 'SELECT NULL FROM summary WHERE maxsize > 1024;') ORDER BY name ASC, size ASC;" +query_vt "SELECT name, size, mtime FROM gufi_vt_pentries('${INDEXROOT}', ${THREADS}, NULL, 'SELECT NULL FROM summary WHERE maxsize > 1024;') ORDER BY name ASC, size ASC;" + +echo "# Paths" +query_vt "SELECT path, epath, fpath, rpath FROM gufi_vt_pentries('${INDEXROOT}', ${THREADS}) WHERE name == '.hidden';" echo "# Missing thread count (not an error)" query_vt "SELECT name FROM gufi_vt_pentries('${INDEXROOT}') ORDER BY name ASC, size ASC;" @@ -126,12 +115,6 @@ query_vt "SELECT name FROM gufi_vt_pentries('${INDEXROOT}') ORDER BY name ASC, s echo "# NULL thread count (not an error)" query_vt "SELECT name FROM gufi_vt_pentries('${INDEXROOT}', NULL) ORDER BY name ASC, size ASC;" -echo "# Place results in a specific directory" -query_vt "SELECT name, size, mtime FROM gufi_vt_pentries('${INDEXROOT}', ${THREADS}, '${ALT_DIR}') ORDER BY name ASC, size ASC;" | sed "s/${ALT_DIR}/alt_dir/g;" - -echo "# Make sure all types work" -query_vt "SELECT rowid, 1, 1.0, 'text', CAST('blob' AS BLOB), NULL FROM gufi_vt_pentries('${INDEXROOT}', ${THREADS}) LIMIT 5;" - set +e echo "# Missing indexroot (error)" query_vt "SELECT name FROM gufi_vt_pentries();" diff --git a/test/unit/googletest/PoolArgs.cpp.in b/test/unit/googletest/PoolArgs.cpp.in index 6c60e5ce6..557649606 100644 --- a/test/unit/googletest/PoolArgs.cpp.in +++ b/test/unit/googletest/PoolArgs.cpp.in @@ -124,6 +124,7 @@ void test_common(PoolArgs *pa) { print.mutex = nullptr; print.outfile = file; print.rows = 0; + print.types = nullptr; // read from the database being processed // no need for WHERE - there should only be 1 table diff --git a/test/unit/googletest/dbutils.cpp.in b/test/unit/googletest/dbutils.cpp.in index 627ebd8c1..7456afbba 100644 --- a/test/unit/googletest/dbutils.cpp.in +++ b/test/unit/googletest/dbutils.cpp.in @@ -815,3 +815,81 @@ TEST(bottomup_collect_treesummary, nullptr) { sll_destroy(&sll, nullptr); EXPECT_EQ(rmdir(dirname), 0); } + +static const char CREATE_TABLE_TEST[] = "CREATE TABLE test(i INT, f FLOAT, t TEXT, b BLOB, n NULL, d DATE);"; + +TEST(get_col_types, have_cols) { + sqlite3 *db = nullptr; + refstr_t str = {}; + int cols = 0; + + ASSERT_EQ(sqlite3_open_v2(SQLITE_MEMORY, &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI, + nullptr), SQLITE_OK); + + ASSERT_EQ(sqlite3_exec(db, CREATE_TABLE_TEST, + nullptr, nullptr, nullptr), SQLITE_OK); + + str.data = "SELECT * FROM test;"; + str.len = strlen(str.data); + + int *types = get_col_types(db, &str, &cols); + ASSERT_NE(types, nullptr); + + const int expected[] = { SQLITE_INTEGER, SQLITE_FLOAT, SQLITE_TEXT, SQLITE_BLOB, SQLITE_NULL, 0}; + EXPECT_EQ((std::size_t) cols, sizeof(expected) / sizeof(expected[0])); + + for(std::size_t i = 0; i < (sizeof(expected) / sizeof(expected[0])); i++) { + EXPECT_EQ(types[i], expected[i]); + } + + free(types); + + sqlite3_close(db); +} + +TEST(get_col_types, no_cols) { + sqlite3 *db = nullptr; + refstr_t str = {}; + int cols = 0; + + ASSERT_EQ(sqlite3_open_v2(SQLITE_MEMORY, &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI, + nullptr), SQLITE_OK); + + ASSERT_EQ(sqlite3_exec(db, CREATE_TABLE_TEST, + nullptr, nullptr, nullptr), SQLITE_OK); + + str.data = "INSERT INTO test (i) VALUES (0);"; + str.len = strlen(str.data); + + int *types = get_col_types(db, &str, &cols); + EXPECT_EQ(types, nullptr); + + sqlite3_close(db); +} + +TEST(get_col_types, bad) { + sqlite3 *db = nullptr; + refstr_t str = {}; + int cols = 0; + + ASSERT_EQ(sqlite3_open_v2(SQLITE_MEMORY, &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI, + nullptr), + SQLITE_OK); + ASSERT_EQ(sqlite3_exec(db, CREATE_TABLE_TEST, + nullptr, nullptr, nullptr), SQLITE_OK); + + EXPECT_EQ(get_col_types(nullptr, &str, &cols), nullptr); + EXPECT_EQ(get_col_types(db, &str, &cols), nullptr); + // &cols == nullptr will break + + str.data = "bad SQL"; + str.len = strlen(str.data); + + int *types = get_col_types(db, &str, &cols); + EXPECT_EQ(types, nullptr); + + sqlite3_close(db); +} diff --git a/test/unit/googletest/print.cpp b/test/unit/googletest/print.cpp index 2188848f4..ca5eca6b7 100644 --- a/test/unit/googletest/print.cpp +++ b/test/unit/googletest/print.cpp @@ -64,13 +64,14 @@ OF SUCH DAMAGE. #include #include +#include #include #include #include "print.h" -static void print_parallel_mutex(pthread_mutex_t *mutex) { +static void print_parallel_mutex_actual(pthread_mutex_t *mutex) { const std::string A = "A"; const std::string BC = "BC"; const std::string D = "D"; @@ -103,6 +104,7 @@ static void print_parallel_mutex(pthread_mutex_t *mutex) { pa.mutex = mutex; pa.outfile = file; pa.rows = 0; + pa.types = nullptr; // A\n is buffered in OutputBuffer and takes up all available space { @@ -155,7 +157,7 @@ static void print_parallel_mutex(pthread_mutex_t *mutex) { // print all data as one row, which is too big for the buffer, so it gets flushed immediately { - const std::string ROW = A + SEP + BC + SEP + D + SEP + SEP +NL; // final separator is for NULL data column, not line end + const std::string ROW = A + SEP + BC + SEP + D + SEP + SEP + NL; // final separator is for NULL data column, not line end EXPECT_EQ(ob.filled, (std::size_t) 0); @@ -171,15 +173,103 @@ static void print_parallel_mutex(pthread_mutex_t *mutex) { } fclose(file); - OutputBuffer_destroy(&ob); delete [] buf; + OutputBuffer_destroy(&ob); } -TEST(print, parallel_w_mutex) { +TEST(print_parallel, mutex) { pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; - print_parallel_mutex(&mutex); + print_parallel_mutex_actual(&mutex); + print_parallel_mutex_actual(nullptr); +} + +static void print_parallel_tlv_actual(const bool use_len) { + const std::string INTEGER = "1"; + const std::string FLOAT = "1.0"; + const std::string TEXT = "text"; + const std::string BLOB = "blob"; + const std::string NULL_ = "NULL"; + const std::string DATE = "date"; + + const char *DATA[] = { + INTEGER.c_str(), + FLOAT.c_str(), + TEXT.c_str(), + BLOB.c_str(), + NULL_.c_str(), + DATE.c_str(), + }; + + const std::size_t COL_COUNT = sizeof(DATA) / sizeof(DATA[0]); + + const int TYPES[] = { + SQLITE_INTEGER, + SQLITE_FLOAT, + SQLITE_TEXT, + SQLITE_BLOB, + SQLITE_NULL, + 0, + }; + + const std::size_t total_len = + INTEGER.size() + FLOAT.size() + TEXT.size() + + BLOB.size() + NULL_.size() + DATE.size() + + 6 - 1 + // column separators + 6 + // 1 octet types + 6 * 4 + // 4 digit lengths + 1 // newline + ; + + struct OutputBuffer ob; + EXPECT_EQ(OutputBuffer_init(&ob, use_len?total_len + 1:1), &ob); + + char *buf = new char[total_len + 1](); + FILE *file = fmemopen(buf, total_len + 1, "w+b"); + ASSERT_NE(file, nullptr); + + PrintArgs pa; + pa.output_buffer = &ob; + pa.delim = '|'; + pa.mutex = nullptr; + pa.outfile = file; + pa.rows = 0; + pa.types = (int *) TYPES; + + EXPECT_EQ(print_parallel(&pa, COL_COUNT, (char **) DATA, nullptr), 0); + EXPECT_EQ(ob.filled, use_len?total_len:0); + EXPECT_EQ(OutputBuffer_flush(&ob, file), use_len?total_len:0); + EXPECT_EQ(fflush(file), 0); + EXPECT_EQ(pa.rows, (std::size_t) 1); + + char *curr = buf; + for(std::size_t i = 0; i < COL_COUNT; i++) { + // type + EXPECT_EQ(*curr, (char) TYPES[i]); + curr++; + + // length + size_t len = 0; + len = (len * 10) + curr[0] - '0'; + len = (len * 10) + curr[1] - '0'; + len = (len * 10) + curr[2] - '0'; + len = (len * 10) + curr[3] - '0'; + curr += 4; + + // value + EXPECT_EQ(len, strlen(DATA[i])); + EXPECT_EQ(std::string(curr, len), DATA[i]); + curr += len; + + // separator + curr++; + } + + fclose(file); + delete [] buf; + OutputBuffer_destroy(&ob); } -TEST(print, parallel_wo_mutex) { - print_parallel_mutex(nullptr); +TEST(print_parallel, tlv) { + print_parallel_tlv_actual(true); + print_parallel_tlv_actual(false); }