A first version of jdbc-copy

README.md

@@ -10,6 +10,7 @@ This project contains some of my small Java based tools.
 * html-unescape ... unescape all html entities included in a file
 * http-cat ... download a http url and print it to stdout
 * i18nbinder ... translate between a set of properties files and an XLS file [i18nbinder/README.md](i18nbinder/README.md)
+* jdbc-copy ... work in progress [jdbc-copy/README.md]/jdbc-copy/README.md)
 * ln ... create links
 * little-proxy ... a forward proxy based on LittleProxy, see [little-proxy/README.md](little-proxy/README.md) for details
 * md5sum ... calculate the md5 hash of a file; emulates the unix command "md5sum"
@@ -43,7 +44,7 @@ Version History
 
 ### 0.5.0 (not yet released)
 
-TBD
+* jdbc-copy: New tool (work in progress)
 
 ### 0.4.0 - 2014-02-08
 

jdbc-copy/README.md

@@ -0,0 +1,26 @@
+jdbc-copy
+=========
+
+Development
+-----------
+
+### Quick Tests
+
+```
+../gradlew copyToOutputLibs
+./scripts/groovy.sh scripts/jdbcCopy.groovy -f etc/h2-from.properties -t etc/h2-to.properties -c etc/tables.conf -a -v
+```
+
+### Thorough Tests
+
+```
+../gradlew dist
+./jdbc-copy*sh -f etc/h2-from.properties -t etc/h2-to.properties -c etc/tables.conf -a -v
+```
+
+Issues
+------
+
+### Adding A Custom JDBC Driver
+
+TBD

jdbc-copy/build.gradle

@@ -0,0 +1,36 @@
+defaultTasks 'sh', 'bat'
+apply plugin: 'java'
+apply plugin: 'eclipse'
+apply plugin: 'application'
+
+mainClassName='jdbcCopy'; // for 'gradle run' and MANIFEST.MF
+ext.groovyScriptName="${projectDir}/scripts/jdbcCopy.groovy";
+
+dependencies {
+  compile 'org.codehaus.groovy:groovy-all:2.2.1'
+  testCompile 'junit:junit:4.11'
+  runtime "commons-cli:commons-cli:1.2"
+  runtime 'org.apache.ant:ant:1.9.3'
+  runtime 'com.h2database:h2:1.3.175'
+}
+
+repositories {
+    mavenCentral()
+}
+
+jar {
+    from {
+      configurations.runtime.collect {
+         it.isDirectory() ? it : zipTree(it).matching {
+            exclude { detail ->
+               detail.getFile().getParentFile().getName().equals("META-INF") && !detail.getFile().getName().equals("dgminfo")
+            }
+         }
+      }
+    }
+    manifest {
+        attributes("Main-Class": mainClassName)
+    }
+}
+
+jar.dependsOn groovyJar

jdbc-copy/etc/h2-from.properties

@@ -0,0 +1,4 @@
+jdbc.driver=org.h2.Driver
+jdbc.url=jdbc:h2:fromH2
+jdbc.username=scott
+jdbc.password=tiger

jdbc-copy/etc/h2-to.properties

@@ -0,0 +1,4 @@
+jdbc.driver=org.h2.Driver
+jdbc.url=jdbc:h2:toH2
+jdbc.username=scott
+jdbc.password=tiger

jdbc-copy/etc/tables.conf

@@ -0,0 +1,4 @@
+tables {
+  dealers = [ id: 'id'        ]
+  options = [ id: 'option_id' ]
+}

jdbc-copy/scripts/groovy.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+D="$(dirname "$0")"
+LIBS="${D}/../build/output/libs"
+exec java -cp "${LIBS}/*" groovy.ui.GroovyMain "$@"

jdbc-copy/scripts/jdbcCopy.groovy

@@ -0,0 +1,221 @@
+import java.sql.SQLException;
+import groovy.sql.Sql;
+import groovy.transform.ToString;
+
+def cli = new CliBuilder(usage: "jdbc-copy [-v][-h] -f from -t to [-C | -i] [-c cfgFile] [-a | tablename1 tablename2 ...]", posix: true);
+cli.with {
+    h   longOpt:   'help',                                                     'Show usage information'
+    v   longOpt:   'verbose',                                                  'Create debug output'
+    a   longOpt:   'all',                                                      'Copy all tables mentioned in cfgFile'
+    'C' longOpt:   'complete',                                                 'Copy all data'
+    'd' longOpt:   'delete',                                                   'Delete records on destination'
+    c   longOpt:   'config',    required: false, args: 1, argName: 'cfgFile',  'Config containing table definitions'
+    f   longOpt:   'from',      required: true,  args: 1, argName: 'fromCfg',  'Config containing source jdbc parameters'
+    t   longOpt:   'to',        required: true,  args: 1, argName: 'toCfg',    'Config containing destination jdbc parameters'
+};
+
+def options = cli.parse(args);
+if (!options) {
+  System.err.println "Unable to parse command line options -> EXIT";
+  System.exit(1);
+}
+
+if (options.h) {
+  cli.usage();
+  System.exit(0);
+}
+
+boolean fVerbose          = options.v;
+boolean fAll              = options.a;
+boolean fComplete         = options.C;
+boolean fDelete           = options.d;
+String fromPropertiesName = options.f;
+Properties fromProperties = new Properties();
+fromProperties.load(new FileInputStream(fromPropertiesName));
+String toPropertiesName   = options.t;
+Properties toProperties = new Properties();
+toProperties.load(new FileInputStream(toPropertiesName));
+
+def tableNames = options.arguments();
+def tables = [];
+
+if (options.c) {
+  String tableConfiguration = options.c;
+  File f = new File(tableConfiguration);
+  if (! f.exists()) {
+    System.err.println "Unable to read file '${tableConfiguration}' -> EXIT";
+    System.exit(1);
+  }
+  ConfigObject configObject = new ConfigSlurper().parse(f.toURI().toURL());
+  tables = configObject.tables;
+  //println configObject.tables.inspect();
+  if (fAll) {
+    tableNames = tables.keySet();
+  }
+} else if (fAll) {
+  System.err.println "You have to specify a cfgFile when using option '-a' -> EXIT";
+  System.exit(1);
+}
+
+if (!tableNames) {
+  System.err.println "No tables to copy -> EXIT";
+  System.exit(1);
+}
+
+def log = {
+  if (fVerbose) {
+    println it;
+  }
+}
+
+Sql fromSql = Sql.newInstance(fromProperties.'jdbc.url', fromProperties.'jdbc.username', fromProperties.'jdbc.password', fromProperties.'jdbc.driver');
+
+Sql toSql = Sql.newInstance(toProperties.'jdbc.url', toProperties.'jdbc.username', toProperties.'jdbc.password', toProperties.'jdbc.driver');
+
+
+tableNames.each { String tableName ->
+  log "Processing table ${tableName}"
+  def constructorArgs = [tableName: tableName, log: log];
+  constructorArgs += tables.get(tableName);
+  def tableDescription = new tableDescription(constructorArgs);
+  tableDescription.copy(fromSql, toSql, fComplete, fDelete);
+}
+System.exit(0);
+
+@ToString(includeNames=true, excludes="log", ignoreNulls=true)
+class tableDescription {
+  String tableName;
+  String id;
+  String sequence;
+  def log;
+
+  public boolean hasId() {
+    return this.id != null;
+  }
+
+  public def getMaxId(Sql sql) {
+    def maxId = null;
+    if (hasId()) {
+      def result = sql.firstRow((String) "select max(${id}) m from ${tableName}");
+      if (result != null && result.m != null) {
+        maxId = result.m;
+      }
+    }
+    return maxId;
+  }
+
+  private def max(def a, def b) {
+    return (a > b) ? a : b;
+  }
+
+  public boolean hasSequence() {
+    return this.sequence != null;
+  }
+
+  void setSequenceForTable(Sql sql, def lastUsedValue) {
+    if (hasSequence()) {
+      try {
+        String drop = "drop sequence ${sequenceName}";
+        log ".. ${tableName} - dropSequence: ${drop}";
+        sql.execute(drop);
+      } catch (SQLException e) {
+        log ".. ${tableName} - unable to update sequence ${sequenceName} - ${e}";
+      }
+      try {
+        String create = "create sequence ${sequenceName} start with ${lastUsedValue+1}";
+        log ".. ${tableName} - createSequence: ${create}";
+        sql.execute(create);
+      } catch (SQLException e) {
+        log ".. ${tableName} - unable to update sequence ${sequenceName} - ${e}";
+      }
+    }
+  }
+
+  long countId(Sql sql, def thisId) {
+    String query = "select count(1) c from ${tableName} where ${id} = ${thisId}";
+    def rows = sql.rows(query);
+    return rows[0].c;
+  }
+
+  void delete(Sql sql, def minId, def maxId) {
+    String delete = "delete from ${tableName}";
+    List<String> criteria = [];
+    if (minId) {
+      criteria << "${id} > ${minId}";
+    }
+    if (maxId) {
+      criteria << "${id} < ${maxId}";
+    }
+    if (criteria) {
+       delete += " where ";
+       delete += criteria.join(" and ");
+    }
+    log ".. ${delete}";
+    sql.execute(delete);
+  }
+
+  public void copy(Sql from, Sql to, boolean fAllRecords, boolean fDeleteRecords) {
+    log "Starting to copy ${tableName}";
+    log ".. ${tableName} - copying all records";
+    def knownMaxId  = -1;  // highest known id
+    long cnt = 0;   // count the number of records found in this table
+    long insertCnt = 0;
+    long updateCnt = 0;
+    boolean thisTableHasId = hasId();
+    String query = "select * from ${tableName}";
+    if (! fAllRecords) {
+      def maxId = getMaxId(from);
+      query = "${query} where ${id} > ${maxId}"; // FIXME: Parameterize!
+    }
+    query += " order by ${id}";
+    log ".. ${query}";
+    def rows = from.rows(query);
+    def previousId = null;
+    for (def row : rows) {
+      boolean fDoInsert = false;
+      boolean fDoUpdate = false;
+      def thisId;
+      if (thisTableHasId) {
+        thisId = row.get(id);
+        knownMaxId = max(knownMaxId, thisId);
+	long thisIdCnt = countId(to, thisId);
+        if (thisIdCnt <= 0) {
+          fDoInsert = true;
+        } else if (thisIdCnt > 1) {
+          log("ERROR - id ${thisId} is not unique within table ${tableName}");
+        } else {
+          fDoUpdate = true;
+        }
+        delete(to, previousId, thisId);
+	previousId = thisId;
+      } else {
+        // ! thisTableHasId
+        fDoInsert = true;
+      }
+      def keySet = row.keySet();
+      if (fDoInsert || fDoUpdate) {
+        String sqlCommand;
+        if (fDoInsert) {
+          ++insertCnt;
+          String insertFieldList = keySet.join(',');
+    	  def colonizedKeySet = keySet.collect{ ":${it}" };
+	  String insertColonizedFieldList = colonizedKeySet.join(',')
+	  sqlCommand = "insert into ${tableName} ( ${insertFieldList} ) values ( ${insertColonizedFieldList} )";
+        } else if (fDoUpdate) {
+	  assert thisId != null;
+          ++updateCnt;
+    	  def updateSet = keySet.collect{ "${it} = :${it}" };
+ 	  sqlCommand = "update ${tableName} set ${updateSet.join(",")} where ${id}=${thisId}";
+        }
+        to.execute(sqlCommand, row);
+      }
+      to.commit(); // commit in order to free the blob objects
+      ++cnt;
+    }
+    delete(to, previousId, null);
+    log ".. ${tableName} - numberOfRecords: ${cnt}, max id: ${knownMaxId}, insertCnt: ${insertCnt}, updateCnt: ${updateCnt}";
+    if (cnt > 0) {
+      setSequenceForTable(to, knownMaxId);
+    }
+  }
+}

jdbc-copy/scripts/test/createV0.groovy

@@ -0,0 +1,57 @@
+import java.sql.SQLException;
+import groovy.sql.Sql;
+
+
+def cli = new CliBuilder(usage: "createV1.groovy [-v][-h] -j jdbcParameters", posix: true);
+cli.with {
+    h   longOpt:   'help',                                                     'Show usage information'
+    v   longOpt:   'verbose',                                                  'Create debug output'
+    j   longOpt:   'jdbc',      required: true,  args: 1, argName: 'jdbcCfg',  'Config containing jdbc parameters'
+};
+
+def options = cli.parse(args);
+if (!options) {
+  System.err.println "Unable to parse command line options -> EXIT";
+  System.exit(1);
+}
+
+if (options.h) {
+  cli.usage();
+  System.exit(0);
+}
+
+boolean fVerbose          = options.v;
+String jdbcPropertiesName = options.j;
+Properties jdbcProperties = new Properties();
+jdbcProperties.load(new FileInputStream(jdbcPropertiesName));
+
+def log = {
+  if (fVerbose) {
+    println it;
+  }
+}
+
+Sql sql = Sql.newInstance(jdbcProperties.'jdbc.url', jdbcProperties.'jdbc.username', jdbcProperties.'jdbc.password', jdbcProperties.'jdbc.driver');
+
+log "Create table DEALERS"
+
+sql.execute('''
+  create table DEALERS (
+    id integer not null primary key,
+    name varchar(200),
+    city varchar(200)
+  )
+''');
+
+log "Create table OPTIONS"
+
+sql.execute('''
+  create table OPTIONS (
+    option_id integer not null primary key,
+    name varchar(200),
+    description varchar(200)
+  )
+''');
+
+sql.commit();
+sql.close();