Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support mysql8 and bump version to 0.22.1-tw-0.8. #11

Open
wants to merge 1 commit into
base: 0.22.1
Choose a base branch
from
Open

Support mysql8 and bump version to 0.22.1-tw-0.8. #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion benchmarks/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
</parent>

<dependencies>
Expand Down
2 changes: 1 addition & 1 deletion cloud/aws-common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion cloud/gcp-common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
3 changes: 3 additions & 0 deletions codestyle/checkstyle-suppressions.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,4 +70,7 @@
<suppress checks="ImportOrder" files="[\\/]target[\\/]generated-test-sources[\\/]" />
<suppress checks="EmptyLineSeparator" files="[\\/]target[\\/]generated-test-sources[\\/]" />
<suppress checks="AvoidStaticImport" files="[\\/]src[\\/](test)[\\/]"/>
<suppress checks=".*" files="extensions-twitter/scribe-emitter/src/main/java/org/apache/druid/emitter/scribe/DruidAdminLogEvent.java"/>
<suppress checks=".*" files="extensions-twitter/scribe-emitter/src/main/java/org/apache/druid/emitter/scribe/DruidIndexingLogEvent.java"/>
<suppress checks=".*" files="extensions-twitter/scribe-emitter/src/main/java/org/apache/druid/emitter/scribe/DruidQueryLogEvent.java"/>
</suppressions>
6 changes: 3 additions & 3 deletions core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<parent>
<artifactId>druid</artifactId>
<groupId>org.apache.druid</groupId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
</parent>

<properties>
Expand Down Expand Up @@ -361,8 +361,8 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<version>${mysql.version}</version>
<scope>test</scope>
</dependency>
Expand Down
71 changes: 62 additions & 9 deletions core/src/main/java/org/apache/druid/utils/ConnectionUriUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

Expand All @@ -44,7 +46,8 @@ public final class ConnectionUriUtils

public static final String POSTGRES_DRIVER = "org.postgresql.Driver";
public static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver";
public static final String MYSQL_NON_REGISTERING_DRIVER = "com.mysql.jdbc.NonRegisteringDriver";
public static final String MYSQL_CONNECTION_URL = "com.mysql.cj.conf.ConnectionUrl";
public static final String MYSQL_HOST_INFO = "com.mysql.cj.conf.HostInfo";
public static final String MARIADB_DRIVER = "org.mariadb.jdbc.Driver";

/**
Expand Down Expand Up @@ -101,7 +104,7 @@ public static Set<String> tryParseJdbcUriParameters(String connectionUri, boolea
}
catch (ClassNotFoundException notFoundMaria2x) {
throw new RuntimeException(
"Failed to find MySQL driver class. Please check the MySQL connector version 5.1.48 is in the classpath",
"Failed to find MySQL driver class. Please check the MySQL connector version 8.2.0 is in the classpath",
notFoundMysql
);
}
Expand Down Expand Up @@ -179,17 +182,67 @@ public static Set<String> tryParsePostgresConnectionUri(String connectionUri)
}

public static Set<String> tryParseMySqlConnectionUri(String connectionUri)
throws ClassNotFoundException, NoSuchMethodException, InstantiationException, IllegalAccessException,
throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException,
InvocationTargetException
{
Class<?> driverClass = Class.forName(MYSQL_NON_REGISTERING_DRIVER);
Method parseUrl = driverClass.getMethod("parseURL", String.class, Properties.class);
// almost the same as postgres, but is an instance level method
Properties properties = (Properties) parseUrl.invoke(driverClass.getConstructor().newInstance(), connectionUri, null);

if (properties == null) {
Class<?> connectionUrlClass = Class.forName(MYSQL_CONNECTION_URL);
Method isConnectionStringSupported = connectionUrlClass.getMethod("acceptsUrl", String.class);
if (!(boolean) isConnectionStringSupported.invoke(connectionUrlClass, connectionUri)) {
throw new IAE("Invalid URL format for MySQL: [%s]", connectionUri);
}
Method getConnectionUrlInstanceMethod = connectionUrlClass.getMethod("getConnectionUrlInstance", String.class, Properties.class);
Object conUrl = getConnectionUrlInstanceMethod.invoke(connectionUrlClass, connectionUri, null);
Method getHostsList = connectionUrlClass.getMethod("getHostsList");
return getKeysFromOptions(getPropertiesFromHosts((List<?>) getHostsList.invoke(conUrl)));
}


private static Properties getPropertiesFromHosts(List<?> hostsList)
throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, ClassNotFoundException
{
Properties properties = new Properties();
Class<?> hostInfoClass = Class.forName(MYSQL_HOST_INFO);
for (Object host : hostsList) {
Method getHostMethod = hostInfoClass.getMethod("getHost");
String hostName = (String) getHostMethod.invoke(host);
if (hostName != null) {
properties.setProperty("HOST", hostName);
}

Method getPortMethod = hostInfoClass.getMethod("getPort");
Integer port = (Integer) getPortMethod.invoke(host);
if (port != null) {
properties.setProperty("PORT", String.valueOf(port));
}

Method getUserMethod = hostInfoClass.getMethod("getUser");
String user = (String) getUserMethod.invoke(host);
if (user != null) {
properties.setProperty("user", user);
}

Method getPasswordMethod = hostInfoClass.getMethod("getPassword");
String password = (String) getPasswordMethod.invoke(host);
if (password != null) {
properties.setProperty("password", password);
}

Method getHostPropertiesMethod = hostInfoClass.getMethod("getHostProperties");
Map<String, String> hostProperties =
(Map<String, String>) getHostPropertiesMethod.invoke(host);
if (hostProperties != null) {
for (Map.Entry<String, String> entry : hostProperties.entrySet()) {
if (entry.getKey() != null && entry.getValue() != null) {
properties.setProperty(entry.getKey(), entry.getValue());
}
}
}
}
return properties;
}

private static Set<String> getKeysFromOptions(Properties properties)
{
Set<String> keys = Sets.newHashSetWithExpectedSize(properties.size());
properties.forEach((k, v) -> keys.add((String) k));
return keys;
Expand Down
11 changes: 2 additions & 9 deletions core/src/test/java/org/apache/druid/utils/ConnectionUriUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ public void testTryParses()

props = ConnectionUriUtils.tryParseJdbcUriParameters(MYSQL_URI, false);
// though this would be 4 if mysql wasn't loaded in classpath because it would fall back to mariadb
Assert.assertEquals(9, props.size());
Assert.assertEquals(6, props.size());

props = ConnectionUriUtils.tryParseJdbcUriParameters(MARIA_URI, false);
Assert.assertEquals(4, props.size());
Expand All @@ -129,13 +129,6 @@ public void tryParseInvalidPostgres()
ConnectionUriUtils.tryParseJdbcUriParameters("jdbc:postgresql://bad:1234&param", true);
}

@Test
public void tryParseInvalidMySql()
{
expectedException.expect(IAE.class);
ConnectionUriUtils.tryParseJdbcUriParameters("jdbc:mysql:/bad", true);
}

@Test
public void testMySqlFallbackMySqlMaria2x()
{
Expand Down Expand Up @@ -205,7 +198,7 @@ public void testMySQLDriver() throws Exception
Set<String> props = ConnectionUriUtils.tryParseMySqlConnectionUri(MYSQL_URI);
// mysql actually misses 'keyonly', but spits out several keys that are not actually uri parameters
// DBNAME, HOST, PORT, HOST.1, PORT.1, NUM_HOSTS
Assert.assertEquals(9, props.size());
Assert.assertEquals(6, props.size());
Assert.assertTrue(props.contains("user"));
Assert.assertTrue(props.contains("password"));
Assert.assertTrue(props.contains("otherOptions"));
Expand Down
2 changes: 1 addition & 1 deletion distribution/bin/generate-binary-license.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ def generate_license(apache_license_v2, license_yaml):
# Print Apache license first.
print_outfile(apache_license_v2)
with open(license_yaml, encoding='utf-8') as registry_file:
licenses_list = list(yaml.load_all(registry_file))
licenses_list = list(yaml.load_all(registry_file, Loader=yaml.FullLoader))

# Group licenses by license_name, license_category, and then module.
licenses_map = {}
Expand Down
2 changes: 1 addition & 1 deletion distribution/bin/generate-binary-notice.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def generate_notice(source_notice, dependences_yaml):
# Print Apache license first.
print_outfile(source_notice)
with open(dependences_yaml, encoding='utf-8') as registry_file:
dependencies = list(yaml.load_all(registry_file))
dependencies = list(yaml.load_all(registry_file, Loader=yaml.FullLoader))

# Group dependencies by module
modules_map = defaultdict(list)
Expand Down
8 changes: 4 additions & 4 deletions distribution/docker/Dockerfile.mysql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ FROM $DRUID_RELEASE

WORKDIR /opt/druid/extensions/mysql-metadata-storage

ARG MYSQL_URL=https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.48/mysql-connector-java-5.1.48.jar
ARG MYSQL_JAR=mysql-connector-java-5.1.48.jar
# https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.48/mysql-connector-java-5.1.48.jar.sha1
ARG MYSQL_SHA=9140be77aafa5050bf4bb936d560cbacb5a6b5c1
ARG MYSQL_URL=https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.2.0/mysql-connector-j-8.2.0.jar
ARG MYSQL_JAR=mysql-connector-j-8.2.0.jar
# https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.2.0/mysql-connector-j-8.2.0.jar.sha1
ARG MYSQL_SHA=56d34aea30915904b1c883f1cfae731dd2df6029

RUN wget -q ${MYSQL_URL} \
&& echo "${MYSQL_SHA} ${MYSQL_JAR}" | sha1sum -c \
Expand Down
2 changes: 1 addition & 1 deletion distribution/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
<parent>
<artifactId>druid</artifactId>
<groupId>org.apache.druid</groupId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
</parent>

<dependencies>
Expand Down
2 changes: 1 addition & 1 deletion docs/development/extensions-core/mysql.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ This extension can use Oracle's MySQL JDBC driver which is not included in the D
install it separately. There are a few ways to obtain this library:

- It can be downloaded from the MySQL site at: https://dev.mysql.com/downloads/connector/j/
- It can be fetched from Maven Central at: https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.48/mysql-connector-java-5.1.48.jar
- It can be fetched from Maven Central at: https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.2.0/mysql-connector-j-8.2.0.jar
- It may be available through your package manager, e.g. as `libmysql-java` on APT for a Debian-based OS

This fetches the MySQL connector JAR file with a name like `mysql-connector-java-5.1.48.jar`.
Expand Down
2 changes: 1 addition & 1 deletion docs/operations/use_sbt_to_build_fat_jar.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ libraryDependencies ++= Seq(
"com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-smile-provider" % "2.3.0",
"com.fasterxml.jackson.module" % "jackson-module-jaxb-annotations" % "2.3.0",
"com.sun.jersey" % "jersey-servlet" % "1.17.1",
"mysql" % "mysql-connector-java" % "5.1.34",
"mysql" % "mysql-connector-java" % "8.2.0",
"org.scalatest" %% "scalatest" % "2.2.3" % "test",
"org.mockito" % "mockito-core" % "1.10.19" % "test"
)
Expand Down
2 changes: 1 addition & 1 deletion extendedset/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
</parent>

<dependencies>
Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/aliyun-oss-extensions/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/ambari-metrics-emitter/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/cassandra-storage/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/cloudfiles-extensions/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/distinctcount/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/dropwizard-emitter/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/gce-extensions/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/graphite-emitter/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/influx-extensions/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/influxdb-emitter/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/kafka-emitter/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/materialized-view-maintenance/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<artifactId>druid</artifactId>
<groupId>org.apache.druid</groupId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/materialized-view-selection/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<artifactId>druid</artifactId>
<groupId>org.apache.druid</groupId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/momentsketch/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<artifactId>druid</artifactId>
<groupId>org.apache.druid</groupId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/moving-average-query/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.druid</groupId>
<artifactId>druid</artifactId>
<version>0.22.1-tw-0.7</version>
<version>0.22.1-tw-0.8</version>
<relativePath>../../pom.xml</relativePath>
</parent>

Expand Down
Loading