Skip to content

Commit

Permalink
[huaweicloud#1427] Optimized dual-engine config center health check l…
Browse files Browse the repository at this point in the history 
…ogic (huaweicloud#1439)
  • Loading branch information
@chengyouling
chengyouling authored Feb 22, 2025
1 parent b89d5c4 commit 8ce9d82
Show file tree
Hide file tree
Showing 7 changed files with 222 additions and 88 deletions.
4 changes: 2 additions & 2 deletions ...-nacos-config/src/main/java/com/alibaba/cloud/nacos/configdata/NacosConfigDataLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,11 @@ public ConfigData doLoad(ConfigDataLoaderContext context,

private ConfigService getConfigService(ConfigDataLoaderContext context) {
NacosConfigServiceMasterManager configServiceMaster = getBean(context, NacosConfigServiceMasterManager.class);
if (configServiceMaster != null && configServiceMaster.checkServerConnect()) {
if (configServiceMaster != null && configServiceMaster.isNacosServerHealth()) {
return configServiceMaster.getConfigService();
}
NacosConfigServiceStandbyManager configServiceStandby = getBean(context, NacosConfigServiceStandbyManager.class);
if (configServiceStandby != null && configServiceStandby.checkServerConnect()) {
if (configServiceStandby != null && configServiceStandby.isNacosServerHealth()) {
return configServiceStandby.getConfigService();
}
if (configServiceMaster != null) {
Expand Down
16 changes: 13 additions & 3 deletions ...wei-nacos-config/src/main/java/com/alibaba/cloud/nacos/refresh/NacosContextRefresher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,8 @@ private void startSchedulerTask() {

private void masterStandbyServerCheck() {
synchronized (NacosContextRefresher.class) {
if (!NacosConfigConst.STATUS_UP.equals(currentConfigServiceManager.getConfigService().getServerStatus())) {
// both check interface invoking and GRPC connect for fail count.
if (!currentConfigServiceManager.isNacosServerHealth() || !currentConfigServiceManager.isRpcConnectHealth()) {
if (failCount == 2) {
reRegisterNacosListeners();
failCount = 0;
Expand All @@ -157,13 +158,22 @@ private void masterStandbyServerCheck() {

// if current server is not master, check whether the master status is healthy, if status UP, use master server.
if (isRetryMasterServer
&& !currentConfigServiceManager.getServerAddr().equals(nacosConfigManagers.get(0).getServerAddr())
&& nacosConfigManagers.get(0).checkServerConnect()) {
&& !currentConfigServiceManager.isMasterConfigService()
&& getMasterConfigService().isNacosServerHealth()) {
reRegisterNacosListeners();
}
}
}

private NacosConfigManager getMasterConfigService() {
for (NacosConfigManager configManager : nacosConfigManagers) {
if (configManager.isMasterConfigService()) {
return configManager;
}
}
return nacosConfigManagers.get(0);
}

private void reRegisterNacosListeners() {
shutDownConfigService(currentConfigServiceManager);
clearConfigListener();
Expand Down
107 changes: 35 additions & 72 deletions ...g/src/main/java/com/huaweicloud/nacos/config/client/NacosPropertiesFuzzyQueryService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,23 @@

package com.huaweicloud.nacos.config.client;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;

import com.alibaba.cloud.nacos.NacosConfigProperties;
import com.alibaba.nacos.client.auth.impl.NacosAuthLoginConstant;
import com.alibaba.nacos.client.auth.impl.process.HttpLoginProcessor;
import com.alibaba.nacos.client.config.impl.ConfigHttpClientManager;
import com.alibaba.nacos.common.http.HttpRestResult;
import com.alibaba.nacos.common.http.client.NacosRestTemplate;
import com.alibaba.nacos.common.http.param.Header;
import com.alibaba.nacos.common.http.param.Query;
import com.alibaba.nacos.plugin.auth.api.LoginIdentityContext;
import com.huaweicloud.nacos.config.NacosConfigConst;
import com.huaweicloud.nacos.config.manager.ConfigServiceManagerUtils;

Expand All @@ -45,19 +42,11 @@ public class NacosPropertiesFuzzyQueryService {

private static final String NACOS_CONFIG_QUERY_URI = "%s/nacos/v1/cs/configs";

private NacosConfigProperties configProperties;

private Properties properties;

private final Map<String, String> address_token = new ConcurrentHashMap<>();

private long tokenTtl;

private long lastRefreshTime;
private static final NacosPropertiesFuzzyQueryService INSTANCE = new NacosPropertiesFuzzyQueryService();

private long refreshWindow;
private NacosConfigProperties configProperties;

private static final NacosPropertiesFuzzyQueryService INSTANCE = new NacosPropertiesFuzzyQueryService();
private final Map<String, Properties> addressPropertiesMap = new HashMap<>();

private NacosPropertiesFuzzyQueryService() {

Expand All @@ -68,16 +57,24 @@ public static NacosPropertiesFuzzyQueryService getInstance() {
}

public List<PropertyConfigItem> loadRouterProperties() {
try {
String address = chooseAddress();
NacosRestTemplate nacosRestTemplate = ConfigHttpClientManager.getInstance().getNacosRestTemplate();
HttpRestResult<PropertiePageQueryResult> response = nacosRestTemplate.get(buildUrl(address), initHeader(address),
buildQuery(), PropertiePageQueryResult.class);
return response.getData().getPageItems();
} catch (Exception e) {
LOGGER.error("load router properties failed!", e);
return Collections.emptyList();
buildAddressPropertiesMap();
List<String> addresses = getAddresses();
for (String address : addresses) {
try {
NacosRestTemplate nacosRestTemplate = ConfigServiceManagerUtils.getNacosRestTemplate();
String url = ConfigServiceManagerUtils.buildUrl(address, NACOS_CONFIG_QUERY_URI);
Header header = ConfigServiceManagerUtils.initHeader(address, configProperties.getUsername(),
addressPropertiesMap.get(address));
HttpRestResult<PropertiePageQueryResult> response = nacosRestTemplate.get(url, header, buildQuery(),
PropertiePageQueryResult.class);
if (response.getData() != null && !CollectionUtils.isEmpty(response.getData().getPageItems())) {
return response.getData().getPageItems();
}
} catch (Exception e) {
LOGGER.error("load router properties failed!", e);
}
}
return Collections.emptyList();
}

private Query buildQuery() {
Expand All @@ -93,60 +90,26 @@ private Query buildQuery() {
return query;
}

private Header initHeader(String address) {
Header header = Header.newInstance();
if (!StringUtils.isEmpty(configProperties.getUsername())) {
header.addParam(NacosAuthLoginConstant.ACCESSTOKEN, getAccessToken(address));
}
return header;
}

private String buildUrl(String address) {
String prefix = "";
if (!configProperties.getServerAddr().startsWith("http")) {
prefix = "http://";
}
return prefix + String.format(NACOS_CONFIG_QUERY_URI, address);
}

private String getNamespace() {
return StringUtils.isEmpty(configProperties.getNamespace()) ? "" : configProperties.getNamespace();
}

private String chooseAddress() {
properties = configProperties.assembleMasterNacosServerProperties();
if (!configProperties.isMasterStandbyEnabled()) {
return configProperties.getServerAddr();
}
if (ConfigServiceManagerUtils.checkServerConnect(configProperties.getServerAddr())) {
return configProperties.getServerAddr();
private List<String> getAddresses() {
List<String> addresses = new ArrayList<>();
addresses.add(configProperties.getServerAddr());
if (configProperties.isMasterStandbyEnabled() && !StringUtils.isEmpty(configProperties.getStandbyServerAddr())) {
addresses.add(configProperties.getStandbyServerAddr());
}
if (ConfigServiceManagerUtils.checkServerConnect(configProperties.getStandbyServerAddr())) {
properties = configProperties.assembleStandbyNacosServerProperties();
return configProperties.getStandbyServerAddr();
}
return configProperties.getServerAddr();
return addresses;
}

private String getAccessToken(String address) {
if (address_token.get(address) != null
&& (System.currentTimeMillis() - lastRefreshTime) < TimeUnit.SECONDS.toMillis(tokenTtl - refreshWindow)) {
return address_token.get(address);
}
HttpLoginProcessor httpLoginProcessor
= new HttpLoginProcessor(ConfigHttpClientManager.getInstance().getNacosRestTemplate());
properties.setProperty(NacosAuthLoginConstant.SERVER, address);
LoginIdentityContext identityContext = httpLoginProcessor.getResponse(properties);
if (identityContext != null
&& !StringUtils.isEmpty(identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN))) {
tokenTtl = Long.parseLong(identityContext.getParameter(NacosAuthLoginConstant.TOKENTTL));
refreshWindow = tokenTtl / 10;
lastRefreshTime = System.currentTimeMillis();
address_token.put(address, identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN));
return identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN);
private void buildAddressPropertiesMap() {
Properties masterProperties = configProperties.assembleMasterNacosServerProperties();
addressPropertiesMap.put(configProperties.getServerAddr(), masterProperties);
if (!StringUtils.isEmpty(configProperties.getStandbyServerAddr())) {
Properties standbyProperties = configProperties.assembleStandbyNacosServerProperties();
addressPropertiesMap.put(configProperties.getStandbyServerAddr(), standbyProperties);
}
lastRefreshTime = System.currentTimeMillis();
return "";
}

public void setConfigProperties(NacosConfigProperties nacosConfigProperties) {
Expand Down
99 changes: 93 additions & 6 deletions ...-config/src/main/java/com/huaweicloud/nacos/config/manager/ConfigServiceManagerUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,48 @@

package com.huaweicloud.nacos.config.manager;

import static com.alibaba.nacos.api.PropertyKeyConst.USERNAME;

import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.nacos.client.auth.impl.NacosAuthLoginConstant;
import com.alibaba.nacos.client.auth.impl.process.HttpLoginProcessor;
import com.alibaba.nacos.client.config.impl.ConfigHttpClientManager;
import com.alibaba.nacos.common.http.HttpRestResult;
import com.alibaba.nacos.common.http.client.NacosRestTemplate;
import com.alibaba.nacos.common.http.param.Header;
import com.alibaba.nacos.common.http.param.Query;
import com.alibaba.nacos.plugin.auth.api.LoginIdentityContext;

public class ConfigServiceManagerUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(ConfigServiceManagerUtils.class);

private static final Map<String, String> address_token = new ConcurrentHashMap<>();

private static final String NACOS_CONFIG_HEALTH_CHECK_URI = "%s/nacos/v1/console/health/config/readiness";

private static long tokenTtl;

private static long lastRefreshTime;

private static long refreshWindow;

private static volatile NacosRestTemplate nacosRestTemplate;

public static boolean checkServerConnect(String serverAddress) {
if (StringUtils.isEmpty(serverAddress)) {
return false;
Expand All @@ -57,17 +85,76 @@ private static URI parseIpPortFromURI(String uri) {
}

public static NacosConfigManager chooseConfigManager(List<NacosConfigManager> nacosConfigManagers) {
int idx = 0;
while (idx < nacosConfigManagers.size()) {
if (checkServerConnect(nacosConfigManagers.get(idx).getServerAddr())) {
return nacosConfigManagers.get(idx);
for (NacosConfigManager nacosConfigManager : nacosConfigManagers) {
if (nacosConfigManager.isNacosServerHealth()) {
return nacosConfigManager;
}
LOGGER.warn("nacos server [{}] unavailable, choose others.", nacosConfigManagers.get(idx).getServerAddr());
idx++;
LOGGER.warn("nacos server [{}] unavailable, choose others.", nacosConfigManager.getServerAddr());
}
LOGGER.warn("all nacos server unavailable, use master server.");

// if all server unavailable, return master server, ensure listening configuration when service is available again.
return nacosConfigManagers.get(0);
}

public static String buildUrl(String address, String uri) {
String prefix = "";
if (!address.startsWith("http")) {
prefix = "http://";
}
return prefix + String.format(uri, address);
}

public static Header initHeader(String address, String userName, Properties properties) {
Header header = Header.newInstance();
if (!StringUtils.isEmpty(userName)) {
header.addParam(NacosAuthLoginConstant.ACCESSTOKEN, getAccessToken(address, properties));
}
return header;
}

private static String getAccessToken(String address, Properties properties) {
if (address_token.get(address) != null
&& (System.currentTimeMillis() - lastRefreshTime) < TimeUnit.SECONDS.toMillis(tokenTtl - refreshWindow)) {
return address_token.get(address);
}
HttpLoginProcessor httpLoginProcessor
= new HttpLoginProcessor(ConfigHttpClientManager.getInstance().getNacosRestTemplate());
properties.setProperty(NacosAuthLoginConstant.SERVER, address);
LoginIdentityContext identityContext = httpLoginProcessor.getResponse(properties);
if (identityContext != null
&& !StringUtils.isEmpty(identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN))) {
tokenTtl = Long.parseLong(identityContext.getParameter(NacosAuthLoginConstant.TOKENTTL));
refreshWindow = tokenTtl / 10;
lastRefreshTime = System.currentTimeMillis();
address_token.put(address, identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN));
return identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN);
}
lastRefreshTime = System.currentTimeMillis();
return "";
}

public static boolean checkConfigServerHealth(String serverAddr, Properties properties) {
String url = buildUrl(serverAddr, NACOS_CONFIG_HEALTH_CHECK_URI);
Header header = initHeader(serverAddr, properties.getProperty(USERNAME), properties);
try {
HttpRestResult<String> response = getNacosRestTemplate().get(url, header, new Query(), String.class);
if (response.ok()) {
return true;
}
if (response.getCode() == HttpStatus.SC_NOT_FOUND && checkServerConnect(serverAddr)) {
return true;
}
} catch (Exception e) {
LOGGER.error("check server [{}] health failed.", serverAddr);
}
return false;
}

public static NacosRestTemplate getNacosRestTemplate() {
if (nacosRestTemplate == null) {
nacosRestTemplate = ConfigHttpClientManager.getInstance().getNacosRestTemplate();
}
return nacosRestTemplate;
}
}
6 changes: 5 additions & 1 deletion ...i-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,11 @@ public interface NacosConfigManager extends Ordered {

String getServerAddr();

boolean checkServerConnect();
boolean isNacosServerHealth();

void resetConfigService();

boolean isMasterConfigService();

boolean isRpcConnectHealth();
}
Loading

0 comments on commit 8ce9d82

Please sign in to comment.