From 94669fa982ac98d665635116fbbce3e4e68d3ae7 Mon Sep 17 00:00:00 2001 From: An Qi Date: Wed, 24 Jul 2024 13:56:19 +0800 Subject: [PATCH] feat(redis,mongodb): allow to authenticate (#402) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 配置方法 Mongodb 使用 uri 对连接参数进行详细配置，包括鉴权机制，相关语法参见 Mongodb 文档 Redis 使用 username 和 password 两个参数配置用户名和参数，若不设置该选项，则不使用鉴权。 --- conf/config.properties | 2 +- .../tsinghua/iginx/mongodb/MongoDBStorage.java | 17 +++++++++-------- .../edu/tsinghua/iginx/redis/RedisStorage.java | 14 +++++++++++--- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index fc45be4f78..3659ff0fd8 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -39,7 +39,7 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool #storageEngineList=127.0.0.1#5432#relational#engine=postgresql#username=postgres#password=postgres#has_data=false #storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path #storageEngineList=127.0.0.1#6667#parquet#dir=/path/to/your/parquet#dummy_dir=/path/to/your/data#iginx_port=6888#has_data=false#is_read_only=false#thrift_timeout=30000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000#write.buffer.size=104857600#write.batch.size=1048576#compact.permits=16#cache.capacity=1073741824#parquet.block.size=134217728#parquet.page.size=8192#parquet.compression=SNAPPY -#storageEngineList=127.0.0.1#27017#mongodb#has_data=false#schema.sample.size=1000#dummy.sample.size=0 +#storageEngineList=127.0.0.1#27017#mongodb#uri="mongodb://127.0.0.1:27017/?maxPoolSize=200&maxIdleTimeMS=60000&waitQueueTimeoutMS=50000"#has_data=false#schema.sample.size=1000#dummy.sample.size=0 #storageEngineList=127.0.0.1#6667#filesystem#dir=/path/to/your/filesystem#dummy_dir=/path/to/your/data#iginx_port=6888#chunk_size_in_bytes=1048576#memory_pool_size=100#has_data=false#is_read_only=false#thrift_timeout=5000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000 #storageEngineList=127.0.0.1#6379#redis#has_data=false#is_read_only=false#timeout=10000#data_db=1#dummy_db=0 diff --git a/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java index e55c72de5b..5824ecaf3a 100644 --- a/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java @@ -54,10 +54,7 @@ import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import cn.edu.tsinghua.iginx.utils.Pair; -import com.mongodb.MongoBulkWriteException; -import com.mongodb.MongoClientSettings; -import com.mongodb.ServerAddress; -import com.mongodb.WriteError; +import com.mongodb.*; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; @@ -82,6 +79,7 @@ public class MongoDBStorage implements IStorage { private static final int SESSION_POOL_MAX_SIZE = 200; public static final String VALUE_FIELD = "v"; public static final String[] SYSTEM_DBS = new String[] {"admin", "config", "local"}; + public static final String CONNECTION_STRING = "uri"; public static final String SCHEMA_SAMPLE_SIZE = "schema.sample.size"; public static final String QUERY_SAMPLE_SIZE = "dummy.sample.size"; public static final String SCHEMA_SAMPLE_SIZE_DEFAULT = "1000"; @@ -97,6 +95,10 @@ public MongoDBStorage(StorageEngineMeta meta) throws StorageInitializationExcept throw new StorageInitializationException("unexpected database: " + meta.getStorageEngine()); } + String defaultConnection = String.format("mongodb://%s:%d", meta.getIp(), meta.getPort()); + String connectionString = + meta.getExtraParams().getOrDefault(CONNECTION_STRING, defaultConnection); + String sampleSize = meta.getExtraParams().getOrDefault(SCHEMA_SAMPLE_SIZE, SCHEMA_SAMPLE_SIZE_DEFAULT); this.schemaSampleSize = Integer.parseInt(sampleSize); @@ -106,7 +108,7 @@ public MongoDBStorage(StorageEngineMeta meta) throws StorageInitializationExcept this.querySampleSize = Integer.parseInt(querySampleSize); try { - this.client = connect(meta.getIp(), meta.getPort()); + this.client = connect(connectionString); } catch (Exception e) { String message = "fail to connect " + meta.getIp() + ":" + meta.getPort(); LOGGER.error(message, e); @@ -114,17 +116,16 @@ public MongoDBStorage(StorageEngineMeta meta) throws StorageInitializationExcept } } - private MongoClient connect(String ip, int port) { - ServerAddress address = new ServerAddress(ip, port); + private MongoClient connect(String connectionString) { MongoClientSettings settings = MongoClientSettings.builder() - .applyToClusterSettings(builder -> builder.hosts(Collections.singletonList(address))) .applyToConnectionPoolSettings( builder -> builder .maxWaitTime(MAX_WAIT_TIME, TimeUnit.SECONDS) .maxSize(SESSION_POOL_MAX_SIZE) .maxConnectionIdleTime(60, TimeUnit.SECONDS)) + .applyConnectionString(new ConnectionString(connectionString)) .build(); return MongoClients.create(settings); diff --git a/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java index b0928cd656..85754b328d 100644 --- a/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java +++ b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java @@ -78,9 +78,13 @@ public class RedisStorage implements IStorage { private static final String TIMEOUT = "timeout"; + private static final String USERNAME = "username"; + + private static final String PASSWORD = "password"; + private static final String DATA_DB = "data_db"; - private static final String DATA_PASSWORD = "dummy_db"; + private static final String DUMMY_DB = "dummy_db"; private static final int DEFAULT_TIMEOUT = 10000; @@ -103,11 +107,15 @@ public RedisStorage(StorageEngineMeta meta) throws StorageInitializationExceptio Map extraParams = meta.getExtraParams(); int timeout = Integer.parseInt(extraParams.getOrDefault(TIMEOUT, String.valueOf(DEFAULT_TIMEOUT))); - this.jedisPool = new JedisPool(new JedisPoolConfig(), meta.getIp(), meta.getPort(), timeout); + String username = extraParams.get(USERNAME); + String password = extraParams.get(PASSWORD); + this.jedisPool = + new JedisPool( + new JedisPoolConfig(), meta.getIp(), meta.getPort(), timeout, username, password); this.dataDb = Integer.parseInt(extraParams.getOrDefault(DATA_DB, String.valueOf(DEFAULT_DATA_DB))); this.dummyDb = - Integer.parseInt(extraParams.getOrDefault(DATA_PASSWORD, String.valueOf(DEFAULT_DUMMY_DB))); + Integer.parseInt(extraParams.getOrDefault(DUMMY_DB, String.valueOf(DEFAULT_DUMMY_DB))); this.dataPrefix = meta.getDataPrefix(); if (dataDb == dummyDb) { throw new StorageInitializationException("data db and dummy db should not be the same");