From 279f9788329ccfb190c85e81f6d03e897dc578f2 Mon Sep 17 00:00:00 2001 From: Uli Heller Date: Wed, 19 Feb 2014 08:37:46 +0100 Subject: [PATCH] A first version of jdbc-copy --- README.md | 3 +- jdbc-copy/README.md | 26 +++ jdbc-copy/build.gradle | 36 ++++ jdbc-copy/etc/h2-from.properties | 4 + jdbc-copy/etc/h2-to.properties | 4 + jdbc-copy/etc/tables.conf | 4 + jdbc-copy/scripts/groovy.sh | 5 + jdbc-copy/scripts/jdbcCopy.groovy | 221 +++++++++++++++++++++++++ jdbc-copy/scripts/test/createV0.groovy | 57 +++++++ jdbc-copy/scripts/test/createV1.groovy | 64 +++++++ jdbc-copy/test-data/fromH2-V1.h2.db | Bin 0 -> 32768 bytes jdbc-copy/test-data/toH2-V0.h2.db | Bin 0 -> 32768 bytes settings.gradle | 1 + 13 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 jdbc-copy/README.md create mode 100644 jdbc-copy/build.gradle create mode 100644 jdbc-copy/etc/h2-from.properties create mode 100644 jdbc-copy/etc/h2-to.properties create mode 100644 jdbc-copy/etc/tables.conf create mode 100755 jdbc-copy/scripts/groovy.sh create mode 100644 jdbc-copy/scripts/jdbcCopy.groovy create mode 100644 jdbc-copy/scripts/test/createV0.groovy create mode 100644 jdbc-copy/scripts/test/createV1.groovy create mode 100644 jdbc-copy/test-data/fromH2-V1.h2.db create mode 100644 jdbc-copy/test-data/toH2-V0.h2.db diff --git a/README.md b/README.md index b037d23..7d01e58 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ This project contains some of my small Java based tools. * html-unescape ... unescape all html entities included in a file * http-cat ... download a http url and print it to stdout * i18nbinder ... translate between a set of properties files and an XLS file [i18nbinder/README.md](i18nbinder/README.md) +* jdbc-copy ... work in progress [jdbc-copy/README.md]/jdbc-copy/README.md) * ln ... create links * little-proxy ... a forward proxy based on LittleProxy, see [little-proxy/README.md](little-proxy/README.md) for details * md5sum ... calculate the md5 hash of a file; emulates the unix command "md5sum" @@ -43,7 +44,7 @@ Version History ### 0.5.0 (not yet released) -TBD +* jdbc-copy: New tool (work in progress) ### 0.4.0 - 2014-02-08 diff --git a/jdbc-copy/README.md b/jdbc-copy/README.md new file mode 100644 index 0000000..639c5a8 --- /dev/null +++ b/jdbc-copy/README.md @@ -0,0 +1,26 @@ +jdbc-copy +========= + +Development +----------- + +### Quick Tests + +``` +../gradlew copyToOutputLibs +./scripts/groovy.sh scripts/jdbcCopy.groovy -f etc/h2-from.properties -t etc/h2-to.properties -c etc/tables.conf -a -v +``` + +### Thorough Tests + +``` +../gradlew dist +./jdbc-copy*sh -f etc/h2-from.properties -t etc/h2-to.properties -c etc/tables.conf -a -v +``` + +Issues +------ + +### Adding A Custom JDBC Driver + +TBD diff --git a/jdbc-copy/build.gradle b/jdbc-copy/build.gradle new file mode 100644 index 0000000..b1fd9a5 --- /dev/null +++ b/jdbc-copy/build.gradle @@ -0,0 +1,36 @@ +defaultTasks 'sh', 'bat' +apply plugin: 'java' +apply plugin: 'eclipse' +apply plugin: 'application' + +mainClassName='jdbcCopy'; // for 'gradle run' and MANIFEST.MF +ext.groovyScriptName="${projectDir}/scripts/jdbcCopy.groovy"; + +dependencies { + compile 'org.codehaus.groovy:groovy-all:2.2.1' + testCompile 'junit:junit:4.11' + runtime "commons-cli:commons-cli:1.2" + runtime 'org.apache.ant:ant:1.9.3' + runtime 'com.h2database:h2:1.3.175' +} + +repositories { + mavenCentral() +} + +jar { + from { + configurations.runtime.collect { + it.isDirectory() ? it : zipTree(it).matching { + exclude { detail -> + detail.getFile().getParentFile().getName().equals("META-INF") && !detail.getFile().getName().equals("dgminfo") + } + } + } + } + manifest { + attributes("Main-Class": mainClassName) + } +} + +jar.dependsOn groovyJar diff --git a/jdbc-copy/etc/h2-from.properties b/jdbc-copy/etc/h2-from.properties new file mode 100644 index 0000000..ecd231b --- /dev/null +++ b/jdbc-copy/etc/h2-from.properties @@ -0,0 +1,4 @@ +jdbc.driver=org.h2.Driver +jdbc.url=jdbc:h2:fromH2 +jdbc.username=scott +jdbc.password=tiger diff --git a/jdbc-copy/etc/h2-to.properties b/jdbc-copy/etc/h2-to.properties new file mode 100644 index 0000000..4ef42db --- /dev/null +++ b/jdbc-copy/etc/h2-to.properties @@ -0,0 +1,4 @@ +jdbc.driver=org.h2.Driver +jdbc.url=jdbc:h2:toH2 +jdbc.username=scott +jdbc.password=tiger diff --git a/jdbc-copy/etc/tables.conf b/jdbc-copy/etc/tables.conf new file mode 100644 index 0000000..a8c58ef --- /dev/null +++ b/jdbc-copy/etc/tables.conf @@ -0,0 +1,4 @@ +tables { + dealers = [ id: 'id' ] + options = [ id: 'option_id' ] +} diff --git a/jdbc-copy/scripts/groovy.sh b/jdbc-copy/scripts/groovy.sh new file mode 100755 index 0000000..2ca24e3 --- /dev/null +++ b/jdbc-copy/scripts/groovy.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +D="$(dirname "$0")" +LIBS="${D}/../build/output/libs" +exec java -cp "${LIBS}/*" groovy.ui.GroovyMain "$@" diff --git a/jdbc-copy/scripts/jdbcCopy.groovy b/jdbc-copy/scripts/jdbcCopy.groovy new file mode 100644 index 0000000..ea1cc8e --- /dev/null +++ b/jdbc-copy/scripts/jdbcCopy.groovy @@ -0,0 +1,221 @@ +import java.sql.SQLException; +import groovy.sql.Sql; +import groovy.transform.ToString; + +def cli = new CliBuilder(usage: "jdbc-copy [-v][-h] -f from -t to [-C | -i] [-c cfgFile] [-a | tablename1 tablename2 ...]", posix: true); +cli.with { + h longOpt: 'help', 'Show usage information' + v longOpt: 'verbose', 'Create debug output' + a longOpt: 'all', 'Copy all tables mentioned in cfgFile' + 'C' longOpt: 'complete', 'Copy all data' + 'd' longOpt: 'delete', 'Delete records on destination' + c longOpt: 'config', required: false, args: 1, argName: 'cfgFile', 'Config containing table definitions' + f longOpt: 'from', required: true, args: 1, argName: 'fromCfg', 'Config containing source jdbc parameters' + t longOpt: 'to', required: true, args: 1, argName: 'toCfg', 'Config containing destination jdbc parameters' +}; + +def options = cli.parse(args); +if (!options) { + System.err.println "Unable to parse command line options -> EXIT"; + System.exit(1); +} + +if (options.h) { + cli.usage(); + System.exit(0); +} + +boolean fVerbose = options.v; +boolean fAll = options.a; +boolean fComplete = options.C; +boolean fDelete = options.d; +String fromPropertiesName = options.f; +Properties fromProperties = new Properties(); +fromProperties.load(new FileInputStream(fromPropertiesName)); +String toPropertiesName = options.t; +Properties toProperties = new Properties(); +toProperties.load(new FileInputStream(toPropertiesName)); + +def tableNames = options.arguments(); +def tables = []; + +if (options.c) { + String tableConfiguration = options.c; + File f = new File(tableConfiguration); + if (! f.exists()) { + System.err.println "Unable to read file '${tableConfiguration}' -> EXIT"; + System.exit(1); + } + ConfigObject configObject = new ConfigSlurper().parse(f.toURI().toURL()); + tables = configObject.tables; + //println configObject.tables.inspect(); + if (fAll) { + tableNames = tables.keySet(); + } +} else if (fAll) { + System.err.println "You have to specify a cfgFile when using option '-a' -> EXIT"; + System.exit(1); +} + +if (!tableNames) { + System.err.println "No tables to copy -> EXIT"; + System.exit(1); +} + +def log = { + if (fVerbose) { + println it; + } +} + +Sql fromSql = Sql.newInstance(fromProperties.'jdbc.url', fromProperties.'jdbc.username', fromProperties.'jdbc.password', fromProperties.'jdbc.driver'); + +Sql toSql = Sql.newInstance(toProperties.'jdbc.url', toProperties.'jdbc.username', toProperties.'jdbc.password', toProperties.'jdbc.driver'); + + +tableNames.each { String tableName -> + log "Processing table ${tableName}" + def constructorArgs = [tableName: tableName, log: log]; + constructorArgs += tables.get(tableName); + def tableDescription = new tableDescription(constructorArgs); + tableDescription.copy(fromSql, toSql, fComplete, fDelete); +} +System.exit(0); + +@ToString(includeNames=true, excludes="log", ignoreNulls=true) +class tableDescription { + String tableName; + String id; + String sequence; + def log; + + public boolean hasId() { + return this.id != null; + } + + public def getMaxId(Sql sql) { + def maxId = null; + if (hasId()) { + def result = sql.firstRow((String) "select max(${id}) m from ${tableName}"); + if (result != null && result.m != null) { + maxId = result.m; + } + } + return maxId; + } + + private def max(def a, def b) { + return (a > b) ? a : b; + } + + public boolean hasSequence() { + return this.sequence != null; + } + + void setSequenceForTable(Sql sql, def lastUsedValue) { + if (hasSequence()) { + try { + String drop = "drop sequence ${sequenceName}"; + log ".. ${tableName} - dropSequence: ${drop}"; + sql.execute(drop); + } catch (SQLException e) { + log ".. ${tableName} - unable to update sequence ${sequenceName} - ${e}"; + } + try { + String create = "create sequence ${sequenceName} start with ${lastUsedValue+1}"; + log ".. ${tableName} - createSequence: ${create}"; + sql.execute(create); + } catch (SQLException e) { + log ".. ${tableName} - unable to update sequence ${sequenceName} - ${e}"; + } + } + } + + long countId(Sql sql, def thisId) { + String query = "select count(1) c from ${tableName} where ${id} = ${thisId}"; + def rows = sql.rows(query); + return rows[0].c; + } + + void delete(Sql sql, def minId, def maxId) { + String delete = "delete from ${tableName}"; + List criteria = []; + if (minId) { + criteria << "${id} > ${minId}"; + } + if (maxId) { + criteria << "${id} < ${maxId}"; + } + if (criteria) { + delete += " where "; + delete += criteria.join(" and "); + } + log ".. ${delete}"; + sql.execute(delete); + } + + public void copy(Sql from, Sql to, boolean fAllRecords, boolean fDeleteRecords) { + log "Starting to copy ${tableName}"; + log ".. ${tableName} - copying all records"; + def knownMaxId = -1; // highest known id + long cnt = 0; // count the number of records found in this table + long insertCnt = 0; + long updateCnt = 0; + boolean thisTableHasId = hasId(); + String query = "select * from ${tableName}"; + if (! fAllRecords) { + def maxId = getMaxId(from); + query = "${query} where ${id} > ${maxId}"; // FIXME: Parameterize! + } + query += " order by ${id}"; + log ".. ${query}"; + def rows = from.rows(query); + def previousId = null; + for (def row : rows) { + boolean fDoInsert = false; + boolean fDoUpdate = false; + def thisId; + if (thisTableHasId) { + thisId = row.get(id); + knownMaxId = max(knownMaxId, thisId); + long thisIdCnt = countId(to, thisId); + if (thisIdCnt <= 0) { + fDoInsert = true; + } else if (thisIdCnt > 1) { + log("ERROR - id ${thisId} is not unique within table ${tableName}"); + } else { + fDoUpdate = true; + } + delete(to, previousId, thisId); + previousId = thisId; + } else { + // ! thisTableHasId + fDoInsert = true; + } + def keySet = row.keySet(); + if (fDoInsert || fDoUpdate) { + String sqlCommand; + if (fDoInsert) { + ++insertCnt; + String insertFieldList = keySet.join(','); + def colonizedKeySet = keySet.collect{ ":${it}" }; + String insertColonizedFieldList = colonizedKeySet.join(',') + sqlCommand = "insert into ${tableName} ( ${insertFieldList} ) values ( ${insertColonizedFieldList} )"; + } else if (fDoUpdate) { + assert thisId != null; + ++updateCnt; + def updateSet = keySet.collect{ "${it} = :${it}" }; + sqlCommand = "update ${tableName} set ${updateSet.join(",")} where ${id}=${thisId}"; + } + to.execute(sqlCommand, row); + } + to.commit(); // commit in order to free the blob objects + ++cnt; + } + delete(to, previousId, null); + log ".. ${tableName} - numberOfRecords: ${cnt}, max id: ${knownMaxId}, insertCnt: ${insertCnt}, updateCnt: ${updateCnt}"; + if (cnt > 0) { + setSequenceForTable(to, knownMaxId); + } + } +} diff --git a/jdbc-copy/scripts/test/createV0.groovy b/jdbc-copy/scripts/test/createV0.groovy new file mode 100644 index 0000000..4a65834 --- /dev/null +++ b/jdbc-copy/scripts/test/createV0.groovy @@ -0,0 +1,57 @@ +import java.sql.SQLException; +import groovy.sql.Sql; + + +def cli = new CliBuilder(usage: "createV1.groovy [-v][-h] -j jdbcParameters", posix: true); +cli.with { + h longOpt: 'help', 'Show usage information' + v longOpt: 'verbose', 'Create debug output' + j longOpt: 'jdbc', required: true, args: 1, argName: 'jdbcCfg', 'Config containing jdbc parameters' +}; + +def options = cli.parse(args); +if (!options) { + System.err.println "Unable to parse command line options -> EXIT"; + System.exit(1); +} + +if (options.h) { + cli.usage(); + System.exit(0); +} + +boolean fVerbose = options.v; +String jdbcPropertiesName = options.j; +Properties jdbcProperties = new Properties(); +jdbcProperties.load(new FileInputStream(jdbcPropertiesName)); + +def log = { + if (fVerbose) { + println it; + } +} + +Sql sql = Sql.newInstance(jdbcProperties.'jdbc.url', jdbcProperties.'jdbc.username', jdbcProperties.'jdbc.password', jdbcProperties.'jdbc.driver'); + +log "Create table DEALERS" + +sql.execute(''' + create table DEALERS ( + id integer not null primary key, + name varchar(200), + city varchar(200) + ) +'''); + +log "Create table OPTIONS" + +sql.execute(''' + create table OPTIONS ( + option_id integer not null primary key, + name varchar(200), + description varchar(200) + ) +'''); + +sql.commit(); +sql.close(); diff --git a/jdbc-copy/scripts/test/createV1.groovy b/jdbc-copy/scripts/test/createV1.groovy new file mode 100644 index 0000000..3ed8509 --- /dev/null +++ b/jdbc-copy/scripts/test/createV1.groovy @@ -0,0 +1,64 @@ +import java.sql.SQLException; +import groovy.sql.Sql; + + +def cli = new CliBuilder(usage: "createV1.groovy [-v][-h] -j jdbcParameters", posix: true); +cli.with { + h longOpt: 'help', 'Show usage information' + v longOpt: 'verbose', 'Create debug output' + j longOpt: 'jdbc', required: true, args: 1, argName: 'jdbcCfg', 'Config containing jdbc parameters' +}; + +def options = cli.parse(args); +if (!options) { + System.err.println "Unable to parse command line options -> EXIT"; + System.exit(1); +} + +if (options.h) { + cli.usage(); + System.exit(0); +} + +boolean fVerbose = options.v; +String jdbcPropertiesName = options.j; +Properties jdbcProperties = new Properties(); +jdbcProperties.load(new FileInputStream(jdbcPropertiesName)); + +def log = { + if (fVerbose) { + println it; + } +} + +Sql sql = Sql.newInstance(jdbcProperties.'jdbc.url', jdbcProperties.'jdbc.username', jdbcProperties.'jdbc.password', jdbcProperties.'jdbc.driver'); + +log "Create table DEALERS" + +sql.execute(''' + create table DEALERS ( + id integer not null primary key, + name varchar(200), + city varchar(200) + ) +'''); + +def dealers = sql.dataSet("DEALERS"); +dealers.add(id: 1, name: 'Dealer1', city: 'Stuttgart'); +dealers.add(id: 2, name: 'Dealer2', city: 'Ludwigsburg'); +dealers.add(id: 10, name: 'Dealer10', city: 'Kornwestheim'); + +log "Create table OPTIONS" + +sql.execute(''' + create table OPTIONS ( + option_id integer not null primary key, + name varchar(200), + description varchar(200) + ) +'''); + +def myOptions = sql.dataSet("OPTIONS"); +myOptions.add(option_id: 10, name: 'Option1', description: 'Description #10'); +myOptions.add(option_id: 20, name: 'Option2', description: 'Description #20'); +myOptions.add(option_id: 100, name: 'Option10', description: 'Description #100'); diff --git a/jdbc-copy/test-data/fromH2-V1.h2.db b/jdbc-copy/test-data/fromH2-V1.h2.db new file mode 100644 index 0000000000000000000000000000000000000000..e6d935c079e435ef2c29bfcc06d7ecbb593bbf2c GIT binary patch literal 32768 zcmeI(&2QUe90%~UapEmo)}`%XAi+?LW+L;+LFAzRkoUM?V84{tN_kA4*oCpMi0enCJ0uX=z1RyZP0^2`)^t+e- z^Yklr_9322qz(ZHKmY;|=vM&yf4@?Qg8&2|009U*J_6y;#qINt&u}mj2tWV=5P$## z`Vfe9{Bm?&5FKHE3bNla>=$A7QNUHqD$nIOVK$yU7@j5B zRIlJm@yx;C49TQ>1*hZF8Grkg0>>xg)0g(O(mbZgH0N)vJK1r0t}qo(%=oVd5_LouiMlk#@6TTh_{Syx9Ec4?D=Gl|M zZz93x9_bw7%^)xo0vST0?~GJ+MKTpqm8$cKOiZa#Q;0f88k$LzYpP)yL~YDzdR;PA zt^UDJzhn|&jV zZbXJRkBZ2l}TY1vd)jAz4Sx}Zz^qo0kC|A=oOJ|q| z-_r|k(ePG2NnzGKn67?~&cN5p5h}Y$P1BP2+pPejjppd+^nVPBOrHNy*c zD`Z|W=J)EQyS#P_W{SGBj$1Hpl^3DU!)cCK_3-jJV<#6zVcAzJ_|#*Kmo2l zXkiG47lx6dxg=;kymjn)Fx+BL<=(YWSu&MYRa>Ox;MR&e`Q8%LFn8xVPNuIJib>qj zwknHiO(wZgG4|eTbfLBP|C6WK_fLoah=#2*p})pLpPvmWG3M)W=FJI)9g4LWNkRYu z#}i;C=sJC;dn0cyR%&WBt2OO2%YoI|yV|uyE50?@y{)yk6ni&x@2CnhPS;<#$NJkV z*j{D*Ii>dAgg)Gn=o?;7zGK+FTb!MB_da`nRIbpoi+xhtJ8(6mx=F+RaJ9opU(_QMBm=o0v8Pm?mzq)Rn0VsK> z$t1bvk{&SXYJ;OQ^1w^tIuPJc!^H%T8|0#qxE>C=>F*v4?mW; zj~#AGv8})D6(bf51S9OvlkB}V``Pi95Zna;2tWV=5P-l#6JXjk?bXiuo!dFyl&#H` zjdkZ0xsc;|W^%#3CK$d;0-y0Ni^ScQ$U{wKh99t@T$D3vRQZ)wbGq*Vi_0Y;CN?y~bSK=xlX5 zYs(v**tu0Zj|l$pe=yU5ULXJg2tWV=!yqvE^y_w-Cx;KmY;|fWTk`CeJ(j zKmNjRFr4TG0uX=z1RyXJ0uLP1V-q$Vfq=0?6l!41Rwwb L2tWV=kDCOxWVoaBoqDG@J z>p>F_UNrG)OuVTF{R=$#4|p;0;D2CZjGokkHx0fs9iRgR>X2+Q|H#=bB8I~1A>g~K6yK>z{}fB*y_5MY6$pTGRWP5(vq6FYl= z=Mt$y00Izz00hPr!2Un36yhKN0SG_<0yjq>8hLQEcyoq>kw5?f5P$##ATWl2@a|uB z)DiodiH716{I6mDM~?q4%74T2pG@+PWBe<(c{Z_y2tWV= z5P-lF72xI)oInhHb5ksc;(>U$|K{U9EwDCDNI!I&2d!#Bltg(okv|`vC;7~%;FZMk z`QT-;oE;UMO{A8+?OP?WkWQpl&a|>TQY0mMTU$Y`|8mhsk;p&Mh{1EeOmfdpg@1^JUwEQ( zh!=xEAOw~PiGMs%GpJ%wQd4RTS|^rLZBn9c5>2-V-BnG?B&xQl8!g3Bb**EvsFsp% z>eWtNv6M`Vy*o*@sn@niRozlG%ju}?G@GJxsiwEu1~pA~Yj$zaW_M)Knqut5vayC* zuTw2c67jd(R8*}_cdyRD3G3LIw3N1!jr|AFwTlvFVHqb?k+8sQhAhRunoyb+HI9cp zdZ4TyQR;P4(>2pF6nl_Y0-eToill9*EjC!PP4_bFl|^FGhgp`TGtBd^>4tY$c&C_V zFt58XL-)Bl1J7NqP&r8|#Y*C9|0Q7yJJW7Y#K?MJBex6TOqH~C(+wX~NJB9jr}Z*i zUMB^!#9TUl3uaK|7gmayaRRarScQB(MunKyDG%9G9%_Be!jSJXfJfIY4Ef^1Ff%Nd zB&&xHeb<9ICyK$eP%hgI=fegwrJ{Eg9qI@r%SQhVR%QS+#Fkf>m%0RQNi{q>&+>1`XcP% zj>O-0@8mg#o$2E2tS9~K{gH05vx|LFJ3Vk|N{f=06rQ&OjImx8uLa{4J`|?3~ zO_C4S_oY&?SdmNnrDCDfl`2wK?j02Sm2O2UmDb9|a;eunkk<~^dWHT*x!dce*#V=a zY9gDdFUB~JIEUfuoJlI48FkKIV;)N?D{O*a9WlXPv=NUK!|$%j#mADtW5*yR;i*G= z#Srkpe}4-ndO-jJ5P$##f+!%|bN2r`*#Cn#d&n6A5P$##AmC3xxa;iyG3@{T{Ll*m z5P$##AP_`>g=gQh)1C@q?#LMe5P$##AOHb>0t<`I{*O=i`Qt+`2tWV=5P(1+1%zju z|NloLfy^FBLjVF0fB*y_;74HLp7Z>V?cWa``al2z5P$##0xBRZI?w;N@cF-hP9V~T q00bZa0SNdIxD}s1?lm8pV82G$cO-V)hMy3C00bZa0SMecf&T#K`o)X@ literal 0 HcmV?d00001 diff --git a/settings.gradle b/settings.gradle index 7005b8f..9e6010b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,6 +5,7 @@ include 'hexdump' include 'html-unescape' include 'http-cat' include 'i18nbinder' +include 'jdbc-copy' include 'ln' include 'md5sum' include 'sha1'