diff --git a/README.md b/README.md index b037d23..7d01e58 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ This project contains some of my small Java based tools. * html-unescape ... unescape all html entities included in a file * http-cat ... download a http url and print it to stdout * i18nbinder ... translate between a set of properties files and an XLS file [i18nbinder/README.md](i18nbinder/README.md) +* jdbc-copy ... work in progress [jdbc-copy/README.md]/jdbc-copy/README.md) * ln ... create links * little-proxy ... a forward proxy based on LittleProxy, see [little-proxy/README.md](little-proxy/README.md) for details * md5sum ... calculate the md5 hash of a file; emulates the unix command "md5sum" @@ -43,7 +44,7 @@ Version History ### 0.5.0 (not yet released) -TBD +* jdbc-copy: New tool (work in progress) ### 0.4.0 - 2014-02-08 diff --git a/jdbc-copy/README.md b/jdbc-copy/README.md new file mode 100644 index 0000000..639c5a8 --- /dev/null +++ b/jdbc-copy/README.md @@ -0,0 +1,26 @@ +jdbc-copy +========= + +Development +----------- + +### Quick Tests + +``` +../gradlew copyToOutputLibs +./scripts/groovy.sh scripts/jdbcCopy.groovy -f etc/h2-from.properties -t etc/h2-to.properties -c etc/tables.conf -a -v +``` + +### Thorough Tests + +``` +../gradlew dist +./jdbc-copy*sh -f etc/h2-from.properties -t etc/h2-to.properties -c etc/tables.conf -a -v +``` + +Issues +------ + +### Adding A Custom JDBC Driver + +TBD diff --git a/jdbc-copy/build.gradle b/jdbc-copy/build.gradle new file mode 100644 index 0000000..b1fd9a5 --- /dev/null +++ b/jdbc-copy/build.gradle @@ -0,0 +1,36 @@ +defaultTasks 'sh', 'bat' +apply plugin: 'java' +apply plugin: 'eclipse' +apply plugin: 'application' + +mainClassName='jdbcCopy'; // for 'gradle run' and MANIFEST.MF +ext.groovyScriptName="${projectDir}/scripts/jdbcCopy.groovy"; + +dependencies { + compile 'org.codehaus.groovy:groovy-all:2.2.1' + testCompile 'junit:junit:4.11' + runtime "commons-cli:commons-cli:1.2" + runtime 'org.apache.ant:ant:1.9.3' + runtime 'com.h2database:h2:1.3.175' +} + +repositories { + mavenCentral() +} + +jar { + from { + configurations.runtime.collect { + it.isDirectory() ? it : zipTree(it).matching { + exclude { detail -> + detail.getFile().getParentFile().getName().equals("META-INF") && !detail.getFile().getName().equals("dgminfo") + } + } + } + } + manifest { + attributes("Main-Class": mainClassName) + } +} + +jar.dependsOn groovyJar diff --git a/jdbc-copy/etc/h2-from.properties b/jdbc-copy/etc/h2-from.properties new file mode 100644 index 0000000..ecd231b --- /dev/null +++ b/jdbc-copy/etc/h2-from.properties @@ -0,0 +1,4 @@ +jdbc.driver=org.h2.Driver +jdbc.url=jdbc:h2:fromH2 +jdbc.username=scott +jdbc.password=tiger diff --git a/jdbc-copy/etc/h2-to.properties b/jdbc-copy/etc/h2-to.properties new file mode 100644 index 0000000..4ef42db --- /dev/null +++ b/jdbc-copy/etc/h2-to.properties @@ -0,0 +1,4 @@ +jdbc.driver=org.h2.Driver +jdbc.url=jdbc:h2:toH2 +jdbc.username=scott +jdbc.password=tiger diff --git a/jdbc-copy/etc/tables.conf b/jdbc-copy/etc/tables.conf new file mode 100644 index 0000000..a8c58ef --- /dev/null +++ b/jdbc-copy/etc/tables.conf @@ -0,0 +1,4 @@ +tables { + dealers = [ id: 'id' ] + options = [ id: 'option_id' ] +} diff --git a/jdbc-copy/scripts/groovy.sh b/jdbc-copy/scripts/groovy.sh new file mode 100755 index 0000000..2ca24e3 --- /dev/null +++ b/jdbc-copy/scripts/groovy.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +D="$(dirname "$0")" +LIBS="${D}/../build/output/libs" +exec java -cp "${LIBS}/*" groovy.ui.GroovyMain "$@" diff --git a/jdbc-copy/scripts/jdbcCopy.groovy b/jdbc-copy/scripts/jdbcCopy.groovy new file mode 100644 index 0000000..ea1cc8e --- /dev/null +++ b/jdbc-copy/scripts/jdbcCopy.groovy @@ -0,0 +1,221 @@ +import java.sql.SQLException; +import groovy.sql.Sql; +import groovy.transform.ToString; + +def cli = new CliBuilder(usage: "jdbc-copy [-v][-h] -f from -t to [-C | -i] [-c cfgFile] [-a | tablename1 tablename2 ...]", posix: true); +cli.with { + h longOpt: 'help', 'Show usage information' + v longOpt: 'verbose', 'Create debug output' + a longOpt: 'all', 'Copy all tables mentioned in cfgFile' + 'C' longOpt: 'complete', 'Copy all data' + 'd' longOpt: 'delete', 'Delete records on destination' + c longOpt: 'config', required: false, args: 1, argName: 'cfgFile', 'Config containing table definitions' + f longOpt: 'from', required: true, args: 1, argName: 'fromCfg', 'Config containing source jdbc parameters' + t longOpt: 'to', required: true, args: 1, argName: 'toCfg', 'Config containing destination jdbc parameters' +}; + +def options = cli.parse(args); +if (!options) { + System.err.println "Unable to parse command line options -> EXIT"; + System.exit(1); +} + +if (options.h) { + cli.usage(); + System.exit(0); +} + +boolean fVerbose = options.v; +boolean fAll = options.a; +boolean fComplete = options.C; +boolean fDelete = options.d; +String fromPropertiesName = options.f; +Properties fromProperties = new Properties(); +fromProperties.load(new FileInputStream(fromPropertiesName)); +String toPropertiesName = options.t; +Properties toProperties = new Properties(); +toProperties.load(new FileInputStream(toPropertiesName)); + +def tableNames = options.arguments(); +def tables = []; + +if (options.c) { + String tableConfiguration = options.c; + File f = new File(tableConfiguration); + if (! f.exists()) { + System.err.println "Unable to read file '${tableConfiguration}' -> EXIT"; + System.exit(1); + } + ConfigObject configObject = new ConfigSlurper().parse(f.toURI().toURL()); + tables = configObject.tables; + //println configObject.tables.inspect(); + if (fAll) { + tableNames = tables.keySet(); + } +} else if (fAll) { + System.err.println "You have to specify a cfgFile when using option '-a' -> EXIT"; + System.exit(1); +} + +if (!tableNames) { + System.err.println "No tables to copy -> EXIT"; + System.exit(1); +} + +def log = { + if (fVerbose) { + println it; + } +} + +Sql fromSql = Sql.newInstance(fromProperties.'jdbc.url', fromProperties.'jdbc.username', fromProperties.'jdbc.password', fromProperties.'jdbc.driver'); + +Sql toSql = Sql.newInstance(toProperties.'jdbc.url', toProperties.'jdbc.username', toProperties.'jdbc.password', toProperties.'jdbc.driver'); + + +tableNames.each { String tableName -> + log "Processing table ${tableName}" + def constructorArgs = [tableName: tableName, log: log]; + constructorArgs += tables.get(tableName); + def tableDescription = new tableDescription(constructorArgs); + tableDescription.copy(fromSql, toSql, fComplete, fDelete); +} +System.exit(0); + +@ToString(includeNames=true, excludes="log", ignoreNulls=true) +class tableDescription { + String tableName; + String id; + String sequence; + def log; + + public boolean hasId() { + return this.id != null; + } + + public def getMaxId(Sql sql) { + def maxId = null; + if (hasId()) { + def result = sql.firstRow((String) "select max(${id}) m from ${tableName}"); + if (result != null && result.m != null) { + maxId = result.m; + } + } + return maxId; + } + + private def max(def a, def b) { + return (a > b) ? a : b; + } + + public boolean hasSequence() { + return this.sequence != null; + } + + void setSequenceForTable(Sql sql, def lastUsedValue) { + if (hasSequence()) { + try { + String drop = "drop sequence ${sequenceName}"; + log ".. ${tableName} - dropSequence: ${drop}"; + sql.execute(drop); + } catch (SQLException e) { + log ".. ${tableName} - unable to update sequence ${sequenceName} - ${e}"; + } + try { + String create = "create sequence ${sequenceName} start with ${lastUsedValue+1}"; + log ".. ${tableName} - createSequence: ${create}"; + sql.execute(create); + } catch (SQLException e) { + log ".. ${tableName} - unable to update sequence ${sequenceName} - ${e}"; + } + } + } + + long countId(Sql sql, def thisId) { + String query = "select count(1) c from ${tableName} where ${id} = ${thisId}"; + def rows = sql.rows(query); + return rows[0].c; + } + + void delete(Sql sql, def minId, def maxId) { + String delete = "delete from ${tableName}"; + List criteria = []; + if (minId) { + criteria << "${id} > ${minId}"; + } + if (maxId) { + criteria << "${id} < ${maxId}"; + } + if (criteria) { + delete += " where "; + delete += criteria.join(" and "); + } + log ".. ${delete}"; + sql.execute(delete); + } + + public void copy(Sql from, Sql to, boolean fAllRecords, boolean fDeleteRecords) { + log "Starting to copy ${tableName}"; + log ".. ${tableName} - copying all records"; + def knownMaxId = -1; // highest known id + long cnt = 0; // count the number of records found in this table + long insertCnt = 0; + long updateCnt = 0; + boolean thisTableHasId = hasId(); + String query = "select * from ${tableName}"; + if (! fAllRecords) { + def maxId = getMaxId(from); + query = "${query} where ${id} > ${maxId}"; // FIXME: Parameterize! + } + query += " order by ${id}"; + log ".. ${query}"; + def rows = from.rows(query); + def previousId = null; + for (def row : rows) { + boolean fDoInsert = false; + boolean fDoUpdate = false; + def thisId; + if (thisTableHasId) { + thisId = row.get(id); + knownMaxId = max(knownMaxId, thisId); + long thisIdCnt = countId(to, thisId); + if (thisIdCnt <= 0) { + fDoInsert = true; + } else if (thisIdCnt > 1) { + log("ERROR - id ${thisId} is not unique within table ${tableName}"); + } else { + fDoUpdate = true; + } + delete(to, previousId, thisId); + previousId = thisId; + } else { + // ! thisTableHasId + fDoInsert = true; + } + def keySet = row.keySet(); + if (fDoInsert || fDoUpdate) { + String sqlCommand; + if (fDoInsert) { + ++insertCnt; + String insertFieldList = keySet.join(','); + def colonizedKeySet = keySet.collect{ ":${it}" }; + String insertColonizedFieldList = colonizedKeySet.join(',') + sqlCommand = "insert into ${tableName} ( ${insertFieldList} ) values ( ${insertColonizedFieldList} )"; + } else if (fDoUpdate) { + assert thisId != null; + ++updateCnt; + def updateSet = keySet.collect{ "${it} = :${it}" }; + sqlCommand = "update ${tableName} set ${updateSet.join(",")} where ${id}=${thisId}"; + } + to.execute(sqlCommand, row); + } + to.commit(); // commit in order to free the blob objects + ++cnt; + } + delete(to, previousId, null); + log ".. ${tableName} - numberOfRecords: ${cnt}, max id: ${knownMaxId}, insertCnt: ${insertCnt}, updateCnt: ${updateCnt}"; + if (cnt > 0) { + setSequenceForTable(to, knownMaxId); + } + } +} diff --git a/jdbc-copy/scripts/test/createV0.groovy b/jdbc-copy/scripts/test/createV0.groovy new file mode 100644 index 0000000..4a65834 --- /dev/null +++ b/jdbc-copy/scripts/test/createV0.groovy @@ -0,0 +1,57 @@ +import java.sql.SQLException; +import groovy.sql.Sql; + + +def cli = new CliBuilder(usage: "createV1.groovy [-v][-h] -j jdbcParameters", posix: true); +cli.with { + h longOpt: 'help', 'Show usage information' + v longOpt: 'verbose', 'Create debug output' + j longOpt: 'jdbc', required: true, args: 1, argName: 'jdbcCfg', 'Config containing jdbc parameters' +}; + +def options = cli.parse(args); +if (!options) { + System.err.println "Unable to parse command line options -> EXIT"; + System.exit(1); +} + +if (options.h) { + cli.usage(); + System.exit(0); +} + +boolean fVerbose = options.v; +String jdbcPropertiesName = options.j; +Properties jdbcProperties = new Properties(); +jdbcProperties.load(new FileInputStream(jdbcPropertiesName)); + +def log = { + if (fVerbose) { + println it; + } +} + +Sql sql = Sql.newInstance(jdbcProperties.'jdbc.url', jdbcProperties.'jdbc.username', jdbcProperties.'jdbc.password', jdbcProperties.'jdbc.driver'); + +log "Create table DEALERS" + +sql.execute(''' + create table DEALERS ( + id integer not null primary key, + name varchar(200), + city varchar(200) + ) +'''); + +log "Create table OPTIONS" + +sql.execute(''' + create table OPTIONS ( + option_id integer not null primary key, + name varchar(200), + description varchar(200) + ) +'''); + +sql.commit(); +sql.close(); diff --git a/jdbc-copy/scripts/test/createV1.groovy b/jdbc-copy/scripts/test/createV1.groovy new file mode 100644 index 0000000..3ed8509 --- /dev/null +++ b/jdbc-copy/scripts/test/createV1.groovy @@ -0,0 +1,64 @@ +import java.sql.SQLException; +import groovy.sql.Sql; + + +def cli = new CliBuilder(usage: "createV1.groovy [-v][-h] -j jdbcParameters", posix: true); +cli.with { + h longOpt: 'help', 'Show usage information' + v longOpt: 'verbose', 'Create debug output' + j longOpt: 'jdbc', required: true, args: 1, argName: 'jdbcCfg', 'Config containing jdbc parameters' +}; + +def options = cli.parse(args); +if (!options) { + System.err.println "Unable to parse command line options -> EXIT"; + System.exit(1); +} + +if (options.h) { + cli.usage(); + System.exit(0); +} + +boolean fVerbose = options.v; +String jdbcPropertiesName = options.j; +Properties jdbcProperties = new Properties(); +jdbcProperties.load(new FileInputStream(jdbcPropertiesName)); + +def log = { + if (fVerbose) { + println it; + } +} + +Sql sql = Sql.newInstance(jdbcProperties.'jdbc.url', jdbcProperties.'jdbc.username', jdbcProperties.'jdbc.password', jdbcProperties.'jdbc.driver'); + +log "Create table DEALERS" + +sql.execute(''' + create table DEALERS ( + id integer not null primary key, + name varchar(200), + city varchar(200) + ) +'''); + +def dealers = sql.dataSet("DEALERS"); +dealers.add(id: 1, name: 'Dealer1', city: 'Stuttgart'); +dealers.add(id: 2, name: 'Dealer2', city: 'Ludwigsburg'); +dealers.add(id: 10, name: 'Dealer10', city: 'Kornwestheim'); + +log "Create table OPTIONS" + +sql.execute(''' + create table OPTIONS ( + option_id integer not null primary key, + name varchar(200), + description varchar(200) + ) +'''); + +def myOptions = sql.dataSet("OPTIONS"); +myOptions.add(option_id: 10, name: 'Option1', description: 'Description #10'); +myOptions.add(option_id: 20, name: 'Option2', description: 'Description #20'); +myOptions.add(option_id: 100, name: 'Option10', description: 'Description #100'); diff --git a/jdbc-copy/test-data/fromH2-V1.h2.db b/jdbc-copy/test-data/fromH2-V1.h2.db new file mode 100644 index 0000000..e6d935c Binary files /dev/null and b/jdbc-copy/test-data/fromH2-V1.h2.db differ diff --git a/jdbc-copy/test-data/toH2-V0.h2.db b/jdbc-copy/test-data/toH2-V0.h2.db new file mode 100644 index 0000000..6bf883c Binary files /dev/null and b/jdbc-copy/test-data/toH2-V0.h2.db differ diff --git a/settings.gradle b/settings.gradle index 7005b8f..9e6010b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,6 +5,7 @@ include 'hexdump' include 'html-unescape' include 'http-cat' include 'i18nbinder' +include 'jdbc-copy' include 'ln' include 'md5sum' include 'sha1'