make querydbs into a python script

much easier to understand
changed:
    removed -N and -V flags
    no longer printing number of rows at end

removed -p, -N, and -V flags from bf.c

config/server.example.in

@@ -69,6 +69,10 @@ Threads=@CORES@
 # single path string
 Query=@CMAKE_INSTALL_PREFIX@/bin/gufi_query
 
+# absolute path to gufi_sqlite3
+# single path string
+Sqlite3=@CMAKE_INSTALL_PREFIX@/bin/gufi_sqlite3
+
 # absolute path to gufi_stat_bin
 # single path string
 Stat=@CMAKE_INSTALL_PREFIX@/bin/gufi_stat_bin

docs/latex/sections/deploy.tex

@@ -121,6 +121,8 @@ \subsubsection{Server}
   \hline
   Query & File Path & The absolute path of \gufiquery. \\
   \hline
+  Sqlite3 & File Path & The absolute path of \gufisqlite. \\
+  \hline
   Stat & File Path & The absolute path of \gufistatbin. \\
   \hline
   IndexRoot & Directory Path & The absolute path of a GUFI tree \\

include/bf.h

@@ -197,9 +197,6 @@ struct input {
    } types;
 
    int  printdir;
-   int  printing;
-   int  printheader;
-   int  printrows;
    int  helped;               /* support parsing of per-app sub-options */
    int  printed_version;
    char delim;

scripts/CMakeLists.txt

@@ -100,6 +100,10 @@ endforeach()
 configure_file(gufi_jail gufi_jail @ONLY)
 install(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/gufi_jail" DESTINATION "${BIN}" COMPONENT Server)
 
+# not prefixed with gufi_ and no client tool
+configure_file(querydbs querydbs @ONLY)
+install(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/querydbs" DESTINATION "${BIN}" COMPONENT Server)
+
 foreach(LIBRARY ${LIBRARIES})
   configure_file("${LIBRARY}" "${LIBRARY}" @ONLY)
 

scripts/gufi_config.py

@@ -121,6 +121,7 @@ def _read_lines(settings, lines, path):
 class Server(Config):
     THREADS      = 'Threads'      # number of threads to use
     QUERY        = 'Query'        # absolute path of gufi_query
+    SQLITE3      = 'Sqlite3'      # absolute path of gufi_sqlite3
     STAT         = 'Stat'         # absolute path of gufi_stat_bin
     INDEXROOT    = 'IndexRoot'    # absolute path of root directory for GUFI to traverse
     OUTPUTBUFFER = 'OutputBuffer' # size of per-thread buffers used to buffer prints
@@ -129,6 +130,7 @@ class Server(Config):
     SETTINGS = {
         THREADS      : gufi_common.get_positive,
         QUERY        : os.path.normpath,
+        SQLITE3      : os.path.normpath,
         STAT         : os.path.normpath,
         INDEXROOT    : os.path.normpath,
         OUTPUTBUFFER : gufi_common.get_non_negative
@@ -148,6 +150,11 @@ def query(self):
         '''return absolute path of gufi_query'''
         return self.config[Server.QUERY]
 
+    @property
+    def sqlite3(self):
+        '''return absolute path of gufi_sqlite3'''
+        return self.config[Server.SQLITE3]
+
     @property
     def stat(self):
         '''return absolute path of gufi_stat_bin'''

scripts/querydbs

@@ -0,0 +1,134 @@
+#!/usr/bin/env @PYTHON_INTERPRETER@
+# This file is part of GUFI, which is part of MarFS, which is released
+# under the BSD license.
+#
+#
+# Copyright (c) 2017, Los Alamos National Security (LANS), LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation and/or
+# other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#
+# From Los Alamos National Security, LLC:
+# LA-CC-15-039
+#
+# Copyright (c) 2017, Los Alamos National Security, LLC All rights reserved.
+# Copyright 2017. Los Alamos National Security, LLC. This software was produced
+# under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
+# Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
+# the U.S. Department of Energy. The U.S. Government has rights to use,
+# reproduce, and distribute this software.  NEITHER THE GOVERNMENT NOR LOS
+# ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
+# ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.  If software is
+# modified to produce derivative works, such modified software should be
+# clearly marked, so as not to confuse it with the version available from
+# LANL.
+
+# THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+
+
+import argparse
+import sys
+import subprocess
+
+import gufi_common
+import gufi_config
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser('querydbs', description='Query GUFI result dbs')
+
+    parser.add_argument('table_name',
+                        help='name of table in database file to attach; also used for view name: \'v<table_name>\'')
+    parser.add_argument('SQL',
+                        help='arbitrary SQL executed on view')
+
+    parser.add_argument('db',
+                        nargs='+',
+                        help='path of source database file(s) to add to view')
+
+    parser.add_argument('-d',
+                        type=gufi_common.get_char,
+                        dest='delim',
+                        default='|',
+                        help='delimiter')
+
+    parser.add_argument('--attach-prefix',
+                        metavar='name',
+                        default='querydbs_prefix',
+                        help='name when attaching dbs')
+
+    parser.add_argument('--verbose', '-V',
+                        action='store_true',
+                        help='Show the equivalent shell command passing raw SQL into gufi_sqlite3')
+
+    return parser.parse_args(argv[1:])
+
+def run(argv, config_path):
+    # find and parse the configuration file first
+    config = gufi_config.Server(config_path)
+
+    args = parse_args(argv)
+
+    attach = [] # ATTACH '<db>' AS <attach_prefix><index>;
+    select = [] # SELECT * FROM <attach_prefix><index>.<table_name>
+
+    for i, db in enumerate(args.db):
+        attach_name = '{}{}'.format(args.attach_prefix, i)
+        attach += ['ATTACH \'{}\' AS {};'.format(db, attach_name)]
+        select += ['SELECT * FROM {}.{}'.format(attach_name, args.table_name)]
+
+    # CREATE TEMP VIEW v<table_name> AS SELECT * FROM <attach_prefix><index>.<table_name> ... ;
+    union_all = 'CREATE TEMP VIEW v{} AS '.format(args.table_name) + ' UNION ALL '.join(select) + ';'
+
+    cmd = [config.sqlite3,'-d', args.delim]
+    sql = attach + [union_all, args.SQL]
+    stdin = '\n'.join(sql)
+
+    if args.verbose:
+        print('(')
+        for line in sql:
+            print('    echo "{}"'.format(line))
+        print(') {} -d "{}"'.format(cmd[0], cmd[2]))
+        sys.stdout.flush()
+
+    query = subprocess.Popen(cmd, stdin=subprocess.PIPE)    # pylint: disable=consider-using-with
+    query.communicate(stdin.encode())                       # block until query finishes
+
+    return query.returncode
+
+if __name__ == '__main__':
+    sys.exit(run(sys.argv, gufi_config.PATH))

src/CMakeLists.txt

@@ -187,7 +187,6 @@ set(C_SOURCES
   gufi_unrollup.c
   parallel_cpr.c
   parallel_rmr.c
-  querydbs.c
 )
 
 # build the C executables

src/bf.c

@@ -150,10 +150,7 @@ void print_help(const char* prog_name,
       case 'H': printf("  -H                     show assigned input values (debugging)"); break;
       case 'v': printf("  -v                     version"); break;
       case 'x': printf("  -x                     index/query xattrs"); break;
-      case 'p': printf("  -p                     print file-names"); break;
       case 'P': printf("  -P                     print directories as they are encountered"); break;
-      case 'N': printf("  -N                     print column-names (header) for DB results"); break;
-      case 'V': printf("  -V                     print column-values (rows) for DB results"); break;
       case 'b': printf("  -b                     build GUFI index tree"); break;
       case 'a': printf("  -a                     AND/OR (SQL query combination)"); break;
       case 'n': printf("  -n <threads>           number of threads"); break;
@@ -205,10 +202,7 @@ void print_help(const char* prog_name,
 
 // DEBUGGING
 void show_input(struct input* in, int retval) {
-   printf("in.printing                 = %d\n",            in->printing);
    printf("in.printdir                 = %d\n",            in->printdir);
-   printf("in.printheader              = %d\n",            in->printheader);
-   printf("in.printrows                = %d\n",            in->printrows);
    printf("in.buildindex               = %d\n",            in->buildindex);
    printf("in.maxthreads               = %zu\n",           in->maxthreads);
    printf("in.delim                    = '%c'\n",          in->delim);
@@ -312,22 +306,10 @@ int parse_cmd_line(int         argc,
          in->process_xattrs = 1;
          break;
 
-      case 'p':               // print file name/path?
-         in->printing = 1;
-         break;
-
       case 'P':               // print dirs?
          in->printdir = 1;
          break;
 
-      case 'N':               // print DB-result column-names?  (i.e. header)
-         in->printheader = 1;
-         break;
-
-      case 'V':               // print DB-result row-values?
-         in->printrows = 1;
-         break;
-
       case 'b':               // build index?
          in->buildindex = 1;
          break;