多集群支持,集群切换

This commit is contained in:
许晓东
2022-01-06 19:31:44 +08:00
parent f5fb2c4f88
commit 6a2d876d50
26 changed files with 317 additions and 209 deletions

View File

@@ -23,4 +23,6 @@ public class KafkaUserDO {
private String password; private String password;
private String updateTime; private String updateTime;
private Long clusterInfoId;
} }

View File

@@ -1,6 +1,7 @@
package com.xuxd.kafka.console.config; package com.xuxd.kafka.console.config;
import java.util.Properties; import java.util.Properties;
import org.apache.kafka.clients.CommonClientConfigs;
/** /**
* kafka-console-ui. * kafka-console-ui.
@@ -12,6 +13,10 @@ public class ContextConfig {
public static final int DEFAULT_REQUEST_TIMEOUT_MS = 5000; public static final int DEFAULT_REQUEST_TIMEOUT_MS = 5000;
private Long clusterInfoId;
private String clusterName;
private String bootstrapServer; private String bootstrapServer;
private int requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS; private int requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS;
@@ -27,7 +32,8 @@ public class ContextConfig {
} }
public int getRequestTimeoutMs() { public int getRequestTimeoutMs() {
return requestTimeoutMs; return properties.containsKey(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG) ?
Integer.parseInt(properties.getProperty(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG)) : requestTimeoutMs;
} }
public void setRequestTimeoutMs(int requestTimeoutMs) { public void setRequestTimeoutMs(int requestTimeoutMs) {
@@ -38,6 +44,22 @@ public class ContextConfig {
return properties; return properties;
} }
public Long getClusterInfoId() {
return clusterInfoId;
}
public void setClusterInfoId(Long clusterInfoId) {
this.clusterInfoId = clusterInfoId;
}
public String getClusterName() {
return clusterName;
}
public void setClusterName(String clusterName) {
this.clusterName = clusterName;
}
public void setProperties(Properties properties) { public void setProperties(Properties properties) {
this.properties = properties; this.properties = properties;
} }

View File

@@ -16,24 +16,8 @@ public class KafkaConfig {
private String bootstrapServer; private String bootstrapServer;
private int requestTimeoutMs;
private String securityProtocol;
private String saslMechanism;
private String saslJaasConfig;
private String adminUsername;
private String adminPassword;
private boolean adminCreate;
private String zookeeperAddr; private String zookeeperAddr;
private boolean enableAcl;
private Properties properties; private Properties properties;
public String getBootstrapServer() { public String getBootstrapServer() {
@@ -44,62 +28,6 @@ public class KafkaConfig {
this.bootstrapServer = bootstrapServer; this.bootstrapServer = bootstrapServer;
} }
public int getRequestTimeoutMs() {
return requestTimeoutMs;
}
public void setRequestTimeoutMs(int requestTimeoutMs) {
this.requestTimeoutMs = requestTimeoutMs;
}
public String getSecurityProtocol() {
return securityProtocol;
}
public void setSecurityProtocol(String securityProtocol) {
this.securityProtocol = securityProtocol;
}
public String getSaslMechanism() {
return saslMechanism;
}
public void setSaslMechanism(String saslMechanism) {
this.saslMechanism = saslMechanism;
}
public String getSaslJaasConfig() {
return saslJaasConfig;
}
public void setSaslJaasConfig(String saslJaasConfig) {
this.saslJaasConfig = saslJaasConfig;
}
public String getAdminUsername() {
return adminUsername;
}
public void setAdminUsername(String adminUsername) {
this.adminUsername = adminUsername;
}
public String getAdminPassword() {
return adminPassword;
}
public void setAdminPassword(String adminPassword) {
this.adminPassword = adminPassword;
}
public boolean isAdminCreate() {
return adminCreate;
}
public void setAdminCreate(boolean adminCreate) {
this.adminCreate = adminCreate;
}
public String getZookeeperAddr() { public String getZookeeperAddr() {
return zookeeperAddr; return zookeeperAddr;
} }
@@ -108,14 +36,6 @@ public class KafkaConfig {
this.zookeeperAddr = zookeeperAddr; this.zookeeperAddr = zookeeperAddr;
} }
public boolean isEnableAcl() {
return enableAcl;
}
public void setEnableAcl(boolean enableAcl) {
this.enableAcl = enableAcl;
}
public Properties getProperties() { public Properties getProperties() {
return properties; return properties;
} }

View File

@@ -36,7 +36,7 @@ public class ConfigController {
this.configService = configService; this.configService = configService;
} }
@GetMapping @GetMapping("/console")
public Object getConfig() { public Object getConfig() {
return ResponseData.create().data(configMap).success(); return ResponseData.create().data(configMap).success();
} }

View File

@@ -27,7 +27,7 @@ import org.springframework.http.MediaType;
* @author xuxd * @author xuxd
* @date 2022-01-05 19:56:25 * @date 2022-01-05 19:56:25
**/ **/
@WebFilter(filterName = "context-set-filter", urlPatterns = {"/*"}) @WebFilter(filterName = "context-set-filter", urlPatterns = {"/acl/*","/user/*","/cluster/*","/config/*","/consumer/*","/message/*","/topic/*","/op/*"})
@Slf4j @Slf4j
public class ContextSetFilter implements Filter { public class ContextSetFilter implements Filter {
@@ -36,6 +36,7 @@ public class ContextSetFilter implements Filter {
{ {
excludes.add("/cluster/info/peek"); excludes.add("/cluster/info/peek");
excludes.add("/cluster/info"); excludes.add("/cluster/info");
excludes.add("/config/console");
} }
@Autowired @Autowired
@@ -49,16 +50,27 @@ public class ContextSetFilter implements Filter {
if (!excludes.contains(uri)) { if (!excludes.contains(uri)) {
String headerId = request.getHeader(Header.ID); String headerId = request.getHeader(Header.ID);
if (StringUtils.isBlank(headerId)) { if (StringUtils.isBlank(headerId)) {
ResponseData failed = ResponseData.create().failed("Cluster id is null."); // ResponseData failed = ResponseData.create().failed("Cluster info is null.");
response.setContentType(MediaType.APPLICATION_JSON_VALUE); ResponseData failed = ResponseData.create().failed("没有集群信息,请先切换集群");
response.getOutputStream().println(ConvertUtil.toJsonString(failed)); response.setContentType(MediaType.APPLICATION_JSON_UTF8_VALUE);
response.getWriter().println(ConvertUtil.toJsonString(failed));
return; return;
} else { } else {
ClusterInfoDO infoDO = clusterInfoMapper.selectById(Long.valueOf(headerId)); ClusterInfoDO infoDO = clusterInfoMapper.selectById(Long.valueOf(headerId));
if (infoDO == null) {
ResponseData failed = ResponseData.create().failed("该集群找不到信息,请切换一个有效集群");
response.setContentType(MediaType.APPLICATION_JSON_UTF8_VALUE);
response.getWriter().println(ConvertUtil.toJsonString(failed));
return;
}
ContextConfig config = new ContextConfig(); ContextConfig config = new ContextConfig();
config.setClusterInfoId(infoDO.getId());
config.setClusterName(infoDO.getClusterName());
config.setBootstrapServer(infoDO.getAddress()); config.setBootstrapServer(infoDO.getAddress());
config.setProperties(ConvertUtil.toProperties(infoDO.getProperties())); if (StringUtils.isNotBlank(infoDO.getProperties())) {
config.setProperties(ConvertUtil.toProperties(infoDO.getProperties()));
}
ContextConfigHolder.CONTEXT_CONFIG.set(config); ContextConfigHolder.CONTEXT_CONFIG.set(config);
} }
} }

View File

@@ -1,9 +1,22 @@
package com.xuxd.kafka.console.schedule; package com.xuxd.kafka.console.schedule;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.xuxd.kafka.console.beans.dos.ClusterInfoDO;
import com.xuxd.kafka.console.beans.dos.KafkaUserDO;
import com.xuxd.kafka.console.config.ContextConfig;
import com.xuxd.kafka.console.config.ContextConfigHolder;
import com.xuxd.kafka.console.dao.ClusterInfoMapper;
import com.xuxd.kafka.console.dao.KafkaUserMapper; import com.xuxd.kafka.console.dao.KafkaUserMapper;
import com.xuxd.kafka.console.utils.ConvertUtil;
import com.xuxd.kafka.console.utils.SaslUtil;
import java.util.ArrayList;
import java.util.List;
import java.util.Set; import java.util.Set;
import kafka.console.KafkaConfigConsole; import kafka.console.KafkaConfigConsole;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -21,25 +34,58 @@ public class KafkaAclSchedule {
private final KafkaConfigConsole configConsole; private final KafkaConfigConsole configConsole;
public KafkaAclSchedule(KafkaUserMapper userMapper, KafkaConfigConsole configConsole) { private final ClusterInfoMapper clusterInfoMapper;
this.userMapper = userMapper;
this.configConsole = configConsole; public KafkaAclSchedule(ObjectProvider<KafkaUserMapper> userMapper,
ObjectProvider<KafkaConfigConsole> configConsole, ObjectProvider<ClusterInfoMapper> clusterInfoMapper) {
this.userMapper = userMapper.getIfAvailable();
this.configConsole = configConsole.getIfAvailable();
this.clusterInfoMapper = clusterInfoMapper.getIfAvailable();
} }
@Scheduled(cron = "${cron.clear-dirty-user}") @Scheduled(cron = "${cron.clear-dirty-user}")
public void clearDirtyKafkaUser() { public void clearDirtyKafkaUser() {
log.info("Start clear dirty data for kafka user from database."); try {
Set<String> userSet = configConsole.getUserList(null); log.info("Start clear dirty data for kafka user from database.");
userMapper.selectList(null).forEach(u -> { List<ClusterInfoDO> clusterInfoDOS = clusterInfoMapper.selectList(null);
if (!userSet.contains(u.getUsername())) { List<Long> clusterInfoIds = new ArrayList<>();
log.info("clear user: {} from database.", u.getUsername()); for (ClusterInfoDO infoDO : clusterInfoDOS) {
try { ContextConfig config = new ContextConfig();
userMapper.deleteById(u.getId()); config.setClusterInfoId(infoDO.getId());
} catch (Exception e) { config.setClusterName(infoDO.getClusterName());
log.error("userMapper.deleteById error, user: " + u, e);
config.setBootstrapServer(infoDO.getAddress());
if (StringUtils.isNotBlank(infoDO.getProperties())) {
config.setProperties(ConvertUtil.toProperties(infoDO.getProperties()));
}
ContextConfigHolder.CONTEXT_CONFIG.set(config);
if (SaslUtil.isEnableSasl() && SaslUtil.isEnableScram()) {
log.info("Start clear cluster: {}", infoDO.getClusterName());
Set<String> userSet = configConsole.getUserList(null);
QueryWrapper<KafkaUserDO> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("cluster_info_id", infoDO.getId());
userMapper.selectList(queryWrapper).forEach(u -> {
if (!userSet.contains(u.getUsername())) {
log.info("clear user: {} from database.", u.getUsername());
try {
userMapper.deleteById(u.getId());
} catch (Exception e) {
log.error("userMapper.deleteById error, user: " + u, e);
}
}
});
clusterInfoIds.add(infoDO.getId());
} }
} }
}); if (CollectionUtils.isNotEmpty(clusterInfoIds)) {
log.info("Clear end."); log.info("Clear the cluster id {}, which not found.", clusterInfoIds);
QueryWrapper<KafkaUserDO> wrapper = new QueryWrapper<>();
wrapper.notIn("cluster_info_id", clusterInfoIds);
userMapper.delete(wrapper);
}
log.info("Clear end.");
} finally {
ContextConfigHolder.CONTEXT_CONFIG.remove();
}
} }
} }

View File

@@ -6,29 +6,37 @@ import com.xuxd.kafka.console.beans.CounterMap;
import com.xuxd.kafka.console.beans.ResponseData; import com.xuxd.kafka.console.beans.ResponseData;
import com.xuxd.kafka.console.beans.dos.KafkaUserDO; import com.xuxd.kafka.console.beans.dos.KafkaUserDO;
import com.xuxd.kafka.console.beans.vo.KafkaUserDetailVO; import com.xuxd.kafka.console.beans.vo.KafkaUserDetailVO;
import com.xuxd.kafka.console.config.KafkaConfig; import com.xuxd.kafka.console.config.ContextConfigHolder;
import com.xuxd.kafka.console.dao.KafkaUserMapper; import com.xuxd.kafka.console.dao.KafkaUserMapper;
import com.xuxd.kafka.console.service.AclService; import com.xuxd.kafka.console.service.AclService;
import com.xuxd.kafka.console.utils.SaslUtil;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Properties;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import kafka.console.KafkaAclConsole; import kafka.console.KafkaAclConsole;
import kafka.console.KafkaConfigConsole; import kafka.console.KafkaConfigConsole;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.admin.ScramMechanism;
import org.apache.kafka.clients.admin.UserScramCredentialsDescription; import org.apache.kafka.clients.admin.UserScramCredentialsDescription;
import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclBinding;
import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclOperation;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.security.auth.SecurityProtocol;
import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.SmartInitializingSingleton;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import scala.Tuple2; import scala.Tuple2;
import static com.xuxd.kafka.console.utils.SaslUtil.isEnableSasl;
import static com.xuxd.kafka.console.utils.SaslUtil.isEnableScram;
/** /**
* kafka-console-ui. * kafka-console-ui.
* *
@@ -37,7 +45,7 @@ import scala.Tuple2;
**/ **/
@Slf4j @Slf4j
@Service @Service
public class AclServiceImpl implements AclService, SmartInitializingSingleton { public class AclServiceImpl implements AclService {
@Autowired @Autowired
private KafkaConfigConsole configConsole; private KafkaConfigConsole configConsole;
@@ -45,9 +53,6 @@ public class AclServiceImpl implements AclService, SmartInitializingSingleton {
@Autowired @Autowired
private KafkaAclConsole aclConsole; private KafkaAclConsole aclConsole;
@Autowired
private KafkaConfig kafkaConfig;
private final KafkaUserMapper kafkaUserMapper; private final KafkaUserMapper kafkaUserMapper;
public AclServiceImpl(ObjectProvider<KafkaUserMapper> kafkaUserMapper) { public AclServiceImpl(ObjectProvider<KafkaUserMapper> kafkaUserMapper) {
@@ -64,15 +69,23 @@ public class AclServiceImpl implements AclService, SmartInitializingSingleton {
} }
@Override public ResponseData addOrUpdateUser(String name, String pass) { @Override public ResponseData addOrUpdateUser(String name, String pass) {
if (!isEnableSasl()) {
return ResponseData.create().failed("Only support SASL protocol.");
}
if (!isEnableScram()) {
return ResponseData.create().failed("Only support SASL_SCRAM.");
}
log.info("add or update user, username: {}, password: {}", name, pass); log.info("add or update user, username: {}, password: {}", name, pass);
if (!configConsole.addOrUpdateUser(name, pass)) { Tuple2<Object, String> tuple2 = configConsole.addOrUpdateUser(name, pass);
if (!(boolean) tuple2._1()) {
log.error("add user to kafka failed."); log.error("add user to kafka failed.");
return ResponseData.create().failed("add user to kafka failed"); return ResponseData.create().failed("add user to kafka failed: " + tuple2._2());
} }
// save user info to database. // save user info to database.
KafkaUserDO userDO = new KafkaUserDO(); KafkaUserDO userDO = new KafkaUserDO();
userDO.setUsername(name); userDO.setUsername(name);
userDO.setPassword(pass); userDO.setPassword(pass);
userDO.setClusterInfoId(ContextConfigHolder.CONTEXT_CONFIG.get().getClusterInfoId());
try { try {
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
map.put("username", name); map.put("username", name);
@@ -86,12 +99,24 @@ public class AclServiceImpl implements AclService, SmartInitializingSingleton {
} }
@Override public ResponseData deleteUser(String name) { @Override public ResponseData deleteUser(String name) {
if (!isEnableSasl()) {
return ResponseData.create().failed("Only support SASL protocol.");
}
if (!isEnableScram()) {
return ResponseData.create().failed("Only support SASL_SCRAM.");
}
log.info("delete user: {}", name); log.info("delete user: {}", name);
Tuple2<Object, String> tuple2 = configConsole.deleteUser(name); Tuple2<Object, String> tuple2 = configConsole.deleteUser(name);
return (boolean) tuple2._1() ? ResponseData.create().success() : ResponseData.create().failed(tuple2._2()); return (boolean) tuple2._1() ? ResponseData.create().success() : ResponseData.create().failed(tuple2._2());
} }
@Override public ResponseData deleteUserAndAuth(String name) { @Override public ResponseData deleteUserAndAuth(String name) {
if (!isEnableSasl()) {
return ResponseData.create().failed("Only support SASL protocol.");
}
if (!isEnableScram()) {
return ResponseData.create().failed("Only support SASL_SCRAM.");
}
log.info("delete user and authority: {}", name); log.info("delete user and authority: {}", name);
AclEntry entry = new AclEntry(); AclEntry entry = new AclEntry();
entry.setPrincipal(name); entry.setPrincipal(name);
@@ -120,7 +145,8 @@ public class AclServiceImpl implements AclService, SmartInitializingSingleton {
Map<String, Object> resultMap = new HashMap<>(); Map<String, Object> resultMap = new HashMap<>();
entryMap.forEach((k, v) -> { entryMap.forEach((k, v) -> {
Map<String, List<AclEntry>> map = v.stream().collect(Collectors.groupingBy(e -> e.getResourceType() + "#" + e.getName())); Map<String, List<AclEntry>> map = v.stream().collect(Collectors.groupingBy(e -> e.getResourceType() + "#" + e.getName()));
if (k.equals(kafkaConfig.getAdminUsername())) { String username = SaslUtil.findUsername(ContextConfigHolder.CONTEXT_CONFIG.get().getProperties().getProperty(SaslConfigs.SASL_JAAS_CONFIG));
if (k.equals(username)) {
Map<String, Object> map2 = new HashMap<>(map); Map<String, Object> map2 = new HashMap<>(map);
Map<String, Object> userMap = new HashMap<>(); Map<String, Object> userMap = new HashMap<>();
userMap.put("role", "admin"); userMap.put("role", "admin");
@@ -133,7 +159,8 @@ public class AclServiceImpl implements AclService, SmartInitializingSingleton {
detailList.values().forEach(u -> { detailList.values().forEach(u -> {
if (!resultMap.containsKey(u.name()) && !u.credentialInfos().isEmpty()) { if (!resultMap.containsKey(u.name()) && !u.credentialInfos().isEmpty()) {
if (!u.name().equals(kafkaConfig.getAdminUsername())) { String username = SaslUtil.findUsername(ContextConfigHolder.CONTEXT_CONFIG.get().getProperties().getProperty(SaslConfigs.SASL_JAAS_CONFIG));
if (!u.name().equals(username)) {
resultMap.put(u.name(), Collections.emptyMap()); resultMap.put(u.name(), Collections.emptyMap());
} else { } else {
Map<String, Object> map2 = new HashMap<>(); Map<String, Object> map2 = new HashMap<>();
@@ -194,27 +221,29 @@ public class AclServiceImpl implements AclService, SmartInitializingSingleton {
} }
Map<String, Object> param = new HashMap<>(); Map<String, Object> param = new HashMap<>();
param.put("username", username); param.put("username", username);
param.put("cluster_info_id", ContextConfigHolder.CONTEXT_CONFIG.get().getClusterInfoId());
List<KafkaUserDO> dos = kafkaUserMapper.selectByMap(param); List<KafkaUserDO> dos = kafkaUserMapper.selectByMap(param);
if (dos.isEmpty()) { if (dos.isEmpty()) {
vo.setConsistencyDescription("Password is null."); vo.setConsistencyDescription("Password is null.");
} else { } else {
vo.setPassword(dos.stream().findFirst().get().getPassword()); vo.setPassword(dos.stream().findFirst().get().getPassword());
// check for consistency. // check for consistency.
boolean consistent = configConsole.isPassConsistent(username, vo.getPassword()); // boolean consistent = configConsole.isPassConsistent(username, vo.getPassword());
vo.setConsistencyDescription(consistent ? "Consistent" : "Password is not consistent."); // vo.setConsistencyDescription(consistent ? "Consistent" : "Password is not consistent.");
vo.setConsistencyDescription("Can not check password consistent.");
} }
return ResponseData.create().data(vo).success(); return ResponseData.create().data(vo).success();
} }
@Override public void afterSingletonsInstantiated() { // @Override public void afterSingletonsInstantiated() {
if (kafkaConfig.isEnableAcl() && kafkaConfig.isAdminCreate()) { // if (kafkaConfig.isEnableAcl() && kafkaConfig.isAdminCreate()) {
log.info("Start create admin user, username: {}, password: {}", kafkaConfig.getAdminUsername(), kafkaConfig.getAdminPassword()); // log.info("Start create admin user, username: {}, password: {}", kafkaConfig.getAdminUsername(), kafkaConfig.getAdminPassword());
boolean done = configConsole.addOrUpdateUserWithZK(kafkaConfig.getAdminUsername(), kafkaConfig.getAdminPassword()); // boolean done = configConsole.addOrUpdateUserWithZK(kafkaConfig.getAdminUsername(), kafkaConfig.getAdminPassword());
if (!done) { // if (!done) {
log.error("Create admin failed."); // log.error("Create admin failed.");
throw new IllegalStateException(); // throw new IllegalStateException();
} // }
} // }
} // }
} }

View File

@@ -0,0 +1,61 @@
package com.xuxd.kafka.console.utils;
import com.xuxd.kafka.console.config.ContextConfigHolder;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.admin.ScramMechanism;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.security.auth.SecurityProtocol;
/**
* kafka-console-ui.
*
* @author xuxd
* @date 2022-01-06 11:07:41
**/
public class SaslUtil {
public static final Pattern JAAS_PATTERN = Pattern.compile("^.*(username=\"(.*)\"[ \t]+).*$");
private SaslUtil() {
}
public static String findUsername(String saslJaasConfig) {
Matcher matcher = JAAS_PATTERN.matcher(saslJaasConfig);
return matcher.find() ? matcher.group(2) : "";
}
public static boolean isEnableSasl() {
Properties properties = ContextConfigHolder.CONTEXT_CONFIG.get().getProperties();
if (!properties.containsKey(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG)) {
return false;
}
String s = properties.getProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG);
SecurityProtocol protocol = SecurityProtocol.valueOf(s);
switch (protocol) {
case SASL_SSL:
case SASL_PLAINTEXT:
return true;
default:
return false;
}
}
public static boolean isEnableScram() {
Properties properties = ContextConfigHolder.CONTEXT_CONFIG.get().getProperties();
if (!properties.containsKey(SaslConfigs.SASL_MECHANISM)) {
return false;
}
String s = properties.getProperty(SaslConfigs.SASL_MECHANISM);
ScramMechanism mechanism = ScramMechanism.fromMechanismName(s);
switch (mechanism) {
case UNKNOWN:
return false;
default:
return true;
}
}
}

View File

@@ -6,23 +6,12 @@ server:
kafka: kafka:
config: config:
# kafka broker地址多个以逗号分隔 # 如果不存在default集群启动的时候默认会把这个加载进来(如果这里配置集群地址了),如果已经存在,则不加载
bootstrap-server: 'localhost:9092' # kafka broker地址多个以逗号分隔不是必须在这里配置也可以启动之后在页面上添加集群信息
request-timeout-ms: 5000 bootstrap-server:
# 服务端是否启用acl如果不启用下面的所有配置都忽略即可只用配置上面的Kafka集群地址就行了 # 集群其它属性配置
enable-acl: false properties:
# 只支持2种安全协议SASL_PLAINTEXT和PLAINTEXT启用acl则设置为SASL_PLAINTEXT不启用acl不需关心这个配置 # request.timeout.ms: 5000
security-protocol: SASL_PLAINTEXT
sasl-mechanism: SCRAM-SHA-256
# 超级管理员用户名在broker上已经配置为超级管理员
admin-username: admin
# 超级管理员密码
admin-password: admin
# 启动自动创建配置的超级管理员用户
admin-create: false
# broker连接的zk地址如果启动自动创建配置的超级管理员用户则必须配置否则忽略
zookeeper-addr: 'localhost:2181'
sasl-jaas-config: org.apache.kafka.common.security.scram.ScramLoginModule required username="${kafka.config.admin-username}" password="${kafka.config.admin-password}";
spring: spring:
application: application:
@@ -46,6 +35,7 @@ spring:
logging: logging:
home: ./ home: ./
# 基于scram方案的acl这里会记录创建的用户密码等信息定时扫描如果集群中已经不存在这些用户就把这些信息从db中清除掉
cron: cron:
# clear-dirty-user: 0 * * * * ? # clear-dirty-user: 0 * * * * ?
clear-dirty-user: 0 0 1 * * ? clear-dirty-user: 0 0 1 * * ?

View File

@@ -3,10 +3,11 @@
-- kafka ACL启用SASL_SCRAM中的用户 -- kafka ACL启用SASL_SCRAM中的用户
CREATE TABLE IF NOT EXISTS T_KAFKA_USER CREATE TABLE IF NOT EXISTS T_KAFKA_USER
( (
ID IDENTITY NOT NULL COMMENT '主键ID', ID IDENTITY NOT NULL COMMENT '主键ID',
USERNAME VARCHAR(50) NOT NULL DEFAULT '' COMMENT '用户名', USERNAME VARCHAR(50) NOT NULL DEFAULT '' COMMENT '用户名',
PASSWORD VARCHAR(50) NOT NULL DEFAULT '' COMMENT '密码', PASSWORD VARCHAR(50) NOT NULL DEFAULT '' COMMENT '密码',
UPDATE_TIME TIMESTAMP NOT NULL DEFAULT NOW() COMMENT '更新时间', UPDATE_TIME TIMESTAMP NOT NULL DEFAULT NOW() COMMENT '更新时间',
CLUSTER_INFO_ID BIGINT NOT NULL COMMENT '集群信息里的集群ID',
PRIMARY KEY (ID), PRIMARY KEY (ID),
UNIQUE (USERNAME) UNIQUE (USERNAME)
); );

View File

@@ -2,9 +2,8 @@ package kafka.console
import java.util.Collections import java.util.Collections
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import com.xuxd.kafka.console.beans.{BrokerNode, ClusterInfo} import com.xuxd.kafka.console.beans.{BrokerNode, ClusterInfo}
import com.xuxd.kafka.console.config.KafkaConfig import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig}
import org.apache.kafka.clients.admin.DescribeClusterResult import org.apache.kafka.clients.admin.DescribeClusterResult
import scala.jdk.CollectionConverters.{CollectionHasAsScala, SetHasAsJava, SetHasAsScala} import scala.jdk.CollectionConverters.{CollectionHasAsScala, SetHasAsJava, SetHasAsScala}
@@ -19,6 +18,7 @@ class ClusterConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConf
def clusterInfo(): ClusterInfo = { def clusterInfo(): ClusterInfo = {
withAdminClientAndCatchError(admin => { withAdminClientAndCatchError(admin => {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val clusterResult: DescribeClusterResult = admin.describeCluster() val clusterResult: DescribeClusterResult = admin.describeCluster()
val clusterInfo = new ClusterInfo val clusterInfo = new ClusterInfo
clusterInfo.setClusterId(clusterResult.clusterId().get(timeoutMs, TimeUnit.MILLISECONDS)) clusterInfo.setClusterId(clusterResult.clusterId().get(timeoutMs, TimeUnit.MILLISECONDS))

View File

@@ -3,8 +3,7 @@ package kafka.console
import java.util import java.util
import java.util.Collections import java.util.Collections
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig}
import com.xuxd.kafka.console.config.KafkaConfig
import kafka.console.ConfigConsole.BrokerLoggerConfigType import kafka.console.ConfigConsole.BrokerLoggerConfigType
import kafka.server.ConfigType import kafka.server.ConfigType
import org.apache.kafka.clients.admin.{AlterConfigOp, Config, ConfigEntry, DescribeConfigsOptions} import org.apache.kafka.clients.admin.{AlterConfigOp, Config, ConfigEntry, DescribeConfigsOptions}
@@ -69,6 +68,7 @@ class ConfigConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConfi
val configResource = new ConfigResource(getResourceTypeAndValidate(entityType, entityName), entityName) val configResource = new ConfigResource(getResourceTypeAndValidate(entityType, entityName), entityName)
val config = Map(configResource -> Collections.singletonList(new AlterConfigOp(entry, opType)).asInstanceOf[util.Collection[AlterConfigOp]]) val config = Map(configResource -> Collections.singletonList(new AlterConfigOp(entry, opType)).asInstanceOf[util.Collection[AlterConfigOp]])
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
admin.incrementalAlterConfigs(config.asJava).all().get(timeoutMs, TimeUnit.MILLISECONDS) admin.incrementalAlterConfigs(config.asJava).all().get(timeoutMs, TimeUnit.MILLISECONDS)
(true, "") (true, "")
}, e => { }, e => {

View File

@@ -1,6 +1,6 @@
package kafka.console package kafka.console
import com.xuxd.kafka.console.config.KafkaConfig import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig}
import org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo import org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo
import org.apache.kafka.clients.admin._ import org.apache.kafka.clients.admin._
import org.apache.kafka.clients.consumer.{ConsumerConfig, OffsetAndMetadata, OffsetResetStrategy} import org.apache.kafka.clients.consumer.{ConsumerConfig, OffsetAndMetadata, OffsetResetStrategy}
@@ -75,6 +75,7 @@ class ConsumerConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaCon
val endOffsets = commitOffsets.keySet.map { topicPartition => val endOffsets = commitOffsets.keySet.map { topicPartition =>
topicPartition -> OffsetSpec.latest topicPartition -> OffsetSpec.latest
}.toMap }.toMap
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
admin.listOffsets(endOffsets.asJava).all().get(timeoutMs, TimeUnit.MILLISECONDS) admin.listOffsets(endOffsets.asJava).all().get(timeoutMs, TimeUnit.MILLISECONDS)
}, e => { }, e => {
log.error("listOffsets error.", e) log.error("listOffsets error.", e)
@@ -166,6 +167,7 @@ class ConsumerConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaCon
def resetPartitionToTargetOffset(groupId: String, partition: TopicPartition, offset: Long): (Boolean, String) = { def resetPartitionToTargetOffset(groupId: String, partition: TopicPartition, offset: Long): (Boolean, String) = {
withAdminClientAndCatchError(admin => { withAdminClientAndCatchError(admin => {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
admin.alterConsumerGroupOffsets(groupId, Map(partition -> new OffsetAndMetadata(offset)).asJava).all().get(timeoutMs, TimeUnit.MILLISECONDS) admin.alterConsumerGroupOffsets(groupId, Map(partition -> new OffsetAndMetadata(offset)).asJava).all().get(timeoutMs, TimeUnit.MILLISECONDS)
(true, "") (true, "")
}, e => { }, e => {
@@ -178,7 +180,7 @@ class ConsumerConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaCon
timestamp: java.lang.Long): (Boolean, String) = { timestamp: java.lang.Long): (Boolean, String) = {
withAdminClientAndCatchError(admin => { withAdminClientAndCatchError(admin => {
val logOffsets = getLogTimestampOffsets(admin, groupId, topicPartitions.asScala, timestamp) val logOffsets = getLogTimestampOffsets(admin, groupId, topicPartitions.asScala, timestamp)
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
admin.alterConsumerGroupOffsets(groupId, logOffsets.asJava).all().get(timeoutMs, TimeUnit.MILLISECONDS) admin.alterConsumerGroupOffsets(groupId, logOffsets.asJava).all().get(timeoutMs, TimeUnit.MILLISECONDS)
(true, "") (true, "")
}, e => { }, e => {
@@ -256,6 +258,7 @@ class ConsumerConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaCon
val timestampOffsets = topicPartitions.map { topicPartition => val timestampOffsets = topicPartitions.map { topicPartition =>
topicPartition -> OffsetSpec.forTimestamp(timestamp) topicPartition -> OffsetSpec.forTimestamp(timestamp)
}.toMap }.toMap
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val offsets = admin.listOffsets( val offsets = admin.listOffsets(
timestampOffsets.asJava, timestampOffsets.asJava,
new ListOffsetsOptions().timeoutMs(timeoutMs) new ListOffsetsOptions().timeoutMs(timeoutMs)
@@ -280,6 +283,7 @@ class ConsumerConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaCon
val endOffsets = topicPartitions.map { topicPartition => val endOffsets = topicPartitions.map { topicPartition =>
topicPartition -> OffsetSpec.latest topicPartition -> OffsetSpec.latest
}.toMap }.toMap
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val offsets = admin.listOffsets( val offsets = admin.listOffsets(
endOffsets.asJava, endOffsets.asJava,
new ListOffsetsOptions().timeoutMs(timeoutMs) new ListOffsetsOptions().timeoutMs(timeoutMs)

View File

@@ -3,9 +3,8 @@ package kafka.console
import java.util import java.util
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.{Collections, List} import java.util.{Collections, List}
import com.xuxd.kafka.console.beans.AclEntry import com.xuxd.kafka.console.beans.AclEntry
import com.xuxd.kafka.console.config.KafkaConfig import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig}
import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.StringUtils
import org.apache.kafka.common.acl._ import org.apache.kafka.common.acl._
import org.apache.kafka.common.resource.{ResourcePattern, ResourcePatternFilter, ResourceType} import org.apache.kafka.common.resource.{ResourcePattern, ResourcePatternFilter, ResourceType}
@@ -58,6 +57,7 @@ class KafkaAclConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaCon
def addAcl(acls: List[AclBinding]): Boolean = { def addAcl(acls: List[AclBinding]): Boolean = {
withAdminClient(adminClient => { withAdminClient(adminClient => {
try { try {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
adminClient.createAcls(acls).all().get(timeoutMs, TimeUnit.MILLISECONDS) adminClient.createAcls(acls).all().get(timeoutMs, TimeUnit.MILLISECONDS)
true true
} catch { } catch {
@@ -100,6 +100,7 @@ class KafkaAclConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaCon
def deleteAcl(entry: AclEntry, allResource: Boolean, allPrincipal: Boolean, allOperation: Boolean): Boolean = { def deleteAcl(entry: AclEntry, allResource: Boolean, allPrincipal: Boolean, allOperation: Boolean): Boolean = {
withAdminClient(adminClient => { withAdminClient(adminClient => {
try { try {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val result = adminClient.deleteAcls(Collections.singleton(entry.toAclBindingFilter(allResource, allPrincipal, allOperation))).all().get(timeoutMs, TimeUnit.MILLISECONDS) val result = adminClient.deleteAcls(Collections.singleton(entry.toAclBindingFilter(allResource, allPrincipal, allOperation))).all().get(timeoutMs, TimeUnit.MILLISECONDS)
log.info("delete acl: {}", result) log.info("delete acl: {}", result)
true true
@@ -113,6 +114,7 @@ class KafkaAclConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaCon
def deleteAcl(filters: util.Collection[AclBindingFilter]): Boolean = { def deleteAcl(filters: util.Collection[AclBindingFilter]): Boolean = {
withAdminClient(adminClient => { withAdminClient(adminClient => {
try { try {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val result = adminClient.deleteAcls(filters).all().get(timeoutMs, TimeUnit.MILLISECONDS) val result = adminClient.deleteAcls(filters).all().get(timeoutMs, TimeUnit.MILLISECONDS)
log.info("delete acl: {}", result) log.info("delete acl: {}", result)
true true

View File

@@ -1,16 +1,16 @@
package kafka.console package kafka.console
import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig}
import kafka.server.ConfigType
import kafka.utils.Implicits.PropertiesOps
import org.apache.kafka.clients.admin._
import org.apache.kafka.common.config.SaslConfigs
import org.apache.kafka.common.security.scram.internals.{ScramCredentialUtils, ScramFormatter}
import java.security.MessageDigest import java.security.MessageDigest
import java.util import java.util
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.{Properties, Set} import java.util.{Properties, Set}
import com.xuxd.kafka.console.config.KafkaConfig
import kafka.server.ConfigType
import kafka.utils.Implicits.PropertiesOps
import org.apache.kafka.clients.admin._
import org.apache.kafka.common.security.scram.internals.{ScramCredentialUtils, ScramFormatter}
import scala.jdk.CollectionConverters.{CollectionHasAsScala, DictionaryHasAsScala, SeqHasAsJava} import scala.jdk.CollectionConverters.{CollectionHasAsScala, DictionaryHasAsScala, SeqHasAsJava}
/** /**
@@ -35,31 +35,32 @@ class KafkaConfigConsole(config: KafkaConfig) extends KafkaConsole(config: Kafka
}).asInstanceOf[util.Map[String, UserScramCredentialsDescription]] }).asInstanceOf[util.Map[String, UserScramCredentialsDescription]]
} }
def addOrUpdateUser(name: String, pass: String): Boolean = { def addOrUpdateUser(name: String, pass: String): (Boolean, String) = {
withAdminClient(adminClient => { withAdminClient(adminClient => {
try { try {
adminClient.alterUserScramCredentials(util.Arrays.asList( val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
new UserScramCredentialUpsertion(name, val mechanisms = ContextConfigHolder.CONTEXT_CONFIG.get().getProperties().getProperty(SaslConfigs.SASL_MECHANISM).split(",").toSeq
new ScramCredentialInfo(ScramMechanism.fromMechanismName(config.getSaslMechanism), defaultIterations), pass))) val scrams = mechanisms.map(m => new UserScramCredentialUpsertion(name,
.all().get(timeoutMs, TimeUnit.MILLISECONDS) new ScramCredentialInfo(ScramMechanism.fromMechanismName(m), defaultIterations), pass))
true adminClient.alterUserScramCredentials(scrams.asInstanceOf[Seq[UserScramCredentialAlteration]].asJava).all().get(timeoutMs, TimeUnit.MILLISECONDS)
(true, "")
} catch { } catch {
case ex: Exception => log.error("addOrUpdateUser error", ex) case ex: Exception => log.error("addOrUpdateUser error", ex)
false (false, ex.getMessage)
} }
}).asInstanceOf[Boolean] }).asInstanceOf[(Boolean, String)]
} }
def addOrUpdateUserWithZK(name: String, pass: String): Boolean = { def addOrUpdateUserWithZK(name: String, pass: String): Boolean = {
withZKClient(adminZkClient => { withZKClient(adminZkClient => {
try { try {
val credential = new ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramMechanism.forMechanismName(config.getSaslMechanism)) val credential = new ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramMechanism.forMechanismName(ContextConfigHolder.CONTEXT_CONFIG.get().getProperties().getProperty(SaslConfigs.SASL_MECHANISM)))
.generateCredential(pass, defaultIterations) .generateCredential(pass, defaultIterations)
val credentialStr = ScramCredentialUtils.credentialToString(credential) val credentialStr = ScramCredentialUtils.credentialToString(credential)
val userConfig: Properties = new Properties() val userConfig: Properties = new Properties()
userConfig.put(config.getSaslMechanism, credentialStr) userConfig.put(ContextConfigHolder.CONTEXT_CONFIG.get().getProperties().getProperty(SaslConfigs.SASL_MECHANISM), credentialStr)
val configs = adminZkClient.fetchEntityConfig(ConfigType.User, name) val configs = adminZkClient.fetchEntityConfig(ConfigType.User, name)
userConfig ++= configs userConfig ++= configs
@@ -101,6 +102,7 @@ class KafkaConfigConsole(config: KafkaConfig) extends KafkaConsole(config: Kafka
// .all().get(timeoutMs, TimeUnit.MILLISECONDS) // .all().get(timeoutMs, TimeUnit.MILLISECONDS)
// all delete // all delete
val userDetail = getUserDetailList(util.Collections.singletonList(name)) val userDetail = getUserDetailList(util.Collections.singletonList(name))
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
userDetail.values().asScala.foreach(u => { userDetail.values().asScala.foreach(u => {
adminClient.alterUserScramCredentials(u.credentialInfos().asScala.map(s => new UserScramCredentialDeletion(u.name(), s.mechanism()) adminClient.alterUserScramCredentials(u.credentialInfos().asScala.map(s => new UserScramCredentialDeletion(u.name(), s.mechanism())
.asInstanceOf[UserScramCredentialAlteration]).toList.asJava) .asInstanceOf[UserScramCredentialAlteration]).toList.asJava)

View File

@@ -2,12 +2,10 @@ package kafka.console
import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig} import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig}
import kafka.zk.{AdminZkClient, KafkaZkClient} import kafka.zk.{AdminZkClient, KafkaZkClient}
import org.apache.kafka.clients.CommonClientConfigs
import org.apache.kafka.clients.admin._ import org.apache.kafka.clients.admin._
import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer, OffsetAndMetadata} import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer, OffsetAndMetadata}
import org.apache.kafka.clients.producer.KafkaProducer import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.config.SaslConfigs
import org.apache.kafka.common.requests.ListOffsetsResponse import org.apache.kafka.common.requests.ListOffsetsResponse
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, StringSerializer} import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, StringSerializer}
import org.apache.kafka.common.utils.Time import org.apache.kafka.common.utils.Time
@@ -25,7 +23,7 @@ import scala.jdk.CollectionConverters.{MapHasAsJava, MapHasAsScala}
* */ * */
class KafkaConsole(config: KafkaConfig) { class KafkaConsole(config: KafkaConfig) {
protected val timeoutMs: Int = config.getRequestTimeoutMs // protected val timeoutMs: Int = config.getRequestTimeoutMs
protected def withAdminClient(f: Admin => Any): Any = { protected def withAdminClient(f: Admin => Any): Any = {
@@ -108,7 +106,7 @@ class KafkaConsole(config: KafkaConfig) {
} }
protected def withTimeoutMs[T <: AbstractOptions[T]](options: T) = { protected def withTimeoutMs[T <: AbstractOptions[T]](options: T) = {
options.timeoutMs(timeoutMs) options.timeoutMs(ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs())
} }
private def createAdminClient(): Admin = { private def createAdminClient(): Admin = {
@@ -120,11 +118,6 @@ class KafkaConsole(config: KafkaConfig) {
props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, ContextConfigHolder.CONTEXT_CONFIG.get().getBootstrapServer()) props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, ContextConfigHolder.CONTEXT_CONFIG.get().getBootstrapServer())
props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()) props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs())
props.putAll(ContextConfigHolder.CONTEXT_CONFIG.get().getProperties()) props.putAll(ContextConfigHolder.CONTEXT_CONFIG.get().getProperties())
if (config.isEnableAcl) {
props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, config.getSecurityProtocol())
props.put(SaslConfigs.SASL_MECHANISM, config.getSaslMechanism())
props.put(SaslConfigs.SASL_JAAS_CONFIG, config.getSaslJaasConfig())
}
props props
} }
} }

View File

@@ -2,7 +2,7 @@ package kafka.console
import com.xuxd.kafka.console.beans.MessageFilter import com.xuxd.kafka.console.beans.MessageFilter
import com.xuxd.kafka.console.beans.enums.FilterType import com.xuxd.kafka.console.beans.enums.FilterType
import com.xuxd.kafka.console.config.KafkaConfig import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig}
import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.StringUtils
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord} import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.clients.producer.ProducerRecord
@@ -27,6 +27,7 @@ class MessageConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConf
var startOffTable: immutable.Map[TopicPartition, Long] = Map.empty var startOffTable: immutable.Map[TopicPartition, Long] = Map.empty
var endOffTable: immutable.Map[TopicPartition, Long] = Map.empty var endOffTable: immutable.Map[TopicPartition, Long] = Map.empty
withAdminClientAndCatchError(admin => { withAdminClientAndCatchError(admin => {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val startTable = KafkaConsole.getLogTimestampOffsets(admin, partitions.asScala.toSeq, startTime, timeoutMs) val startTable = KafkaConsole.getLogTimestampOffsets(admin, partitions.asScala.toSeq, startTime, timeoutMs)
startOffTable = startTable.map(t2 => (t2._1, t2._2.offset())).toMap startOffTable = startTable.map(t2 => (t2._1, t2._2.offset())).toMap
@@ -93,6 +94,7 @@ class MessageConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConf
if (searchNums >= maxNums) { if (searchNums >= maxNums) {
terminate = true terminate = true
} else { } else {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val records = consumer.poll(Duration.ofMillis(timeoutMs)) val records = consumer.poll(Duration.ofMillis(timeoutMs))
if (records.isEmpty) { if (records.isEmpty) {
@@ -189,6 +191,7 @@ class MessageConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConf
var terminate = tpSet.isEmpty var terminate = tpSet.isEmpty
while (!terminate) { while (!terminate) {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val records = consumer.poll(Duration.ofMillis(timeoutMs)) val records = consumer.poll(Duration.ofMillis(timeoutMs))
val tps = new util.HashSet(tpSet).asScala val tps = new util.HashSet(tpSet).asScala
for (tp <- tps) { for (tp <- tps) {

View File

@@ -1,6 +1,6 @@
package kafka.console package kafka.console
import com.xuxd.kafka.console.config.KafkaConfig import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig}
import kafka.admin.ReassignPartitionsCommand import kafka.admin.ReassignPartitionsCommand
import org.apache.kafka.clients.admin.{ElectLeadersOptions, ListPartitionReassignmentsOptions, PartitionReassignment} import org.apache.kafka.clients.admin.{ElectLeadersOptions, ListPartitionReassignmentsOptions, PartitionReassignment}
import org.apache.kafka.clients.consumer.KafkaConsumer import org.apache.kafka.clients.consumer.KafkaConsumer
@@ -34,6 +34,7 @@ class OperationConsole(config: KafkaConfig, topicConsole: TopicConsole,
throw new UnsupportedOperationException("exist consumer client.") throw new UnsupportedOperationException("exist consumer client.")
} }
} }
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val thatGroupDescriptionList = thatAdmin.describeConsumerGroups(searchGroupIds).all().get(timeoutMs, TimeUnit.MILLISECONDS).values() val thatGroupDescriptionList = thatAdmin.describeConsumerGroups(searchGroupIds).all().get(timeoutMs, TimeUnit.MILLISECONDS).values()
if (groupDescriptionList.isEmpty) { if (groupDescriptionList.isEmpty) {
throw new IllegalArgumentException("that consumer group info is null.") throw new IllegalArgumentException("that consumer group info is null.")
@@ -101,6 +102,7 @@ class OperationConsole(config: KafkaConfig, topicConsole: TopicConsole,
thatMinOffset: util.Map[TopicPartition, Long]): (Boolean, String) = { thatMinOffset: util.Map[TopicPartition, Long]): (Boolean, String) = {
val thatAdmin = createAdminClient(props) val thatAdmin = createAdminClient(props)
try { try {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val searchGroupIds = Collections.singleton(groupId) val searchGroupIds = Collections.singleton(groupId)
val groupDescriptionList = consumerConsole.getConsumerGroupList(searchGroupIds) val groupDescriptionList = consumerConsole.getConsumerGroupList(searchGroupIds)
if (groupDescriptionList.isEmpty) { if (groupDescriptionList.isEmpty) {
@@ -178,6 +180,7 @@ class OperationConsole(config: KafkaConfig, topicConsole: TopicConsole,
val thatConsumer = new KafkaConsumer(props, new ByteArrayDeserializer, new ByteArrayDeserializer) val thatConsumer = new KafkaConsumer(props, new ByteArrayDeserializer, new ByteArrayDeserializer)
try { try {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val thisTopicPartitions = consumerConsole.listSubscribeTopics(groupId).get(topic).asScala.sortBy(_.partition()) val thisTopicPartitions = consumerConsole.listSubscribeTopics(groupId).get(topic).asScala.sortBy(_.partition())
val thatTopicPartitionMap = thatAdmin.listConsumerGroupOffsets( val thatTopicPartitionMap = thatAdmin.listConsumerGroupOffsets(
groupId groupId

View File

@@ -1,6 +1,6 @@
package kafka.console package kafka.console
import com.xuxd.kafka.console.config.KafkaConfig import com.xuxd.kafka.console.config.{ContextConfigHolder, KafkaConfig}
import kafka.admin.ReassignPartitionsCommand._ import kafka.admin.ReassignPartitionsCommand._
import kafka.utils.Json import kafka.utils.Json
import org.apache.kafka.clients.admin._ import org.apache.kafka.clients.admin._
@@ -28,6 +28,7 @@ class TopicConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConfig
* @return all topic name set. * @return all topic name set.
*/ */
def getTopicNameList(internal: Boolean = true): Set[String] = { def getTopicNameList(internal: Boolean = true): Set[String] = {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
withAdminClientAndCatchError(admin => admin.listTopics(new ListTopicsOptions().listInternal(internal)).names() withAdminClientAndCatchError(admin => admin.listTopics(new ListTopicsOptions().listInternal(internal)).names()
.get(timeoutMs, TimeUnit.MILLISECONDS), .get(timeoutMs, TimeUnit.MILLISECONDS),
e => { e => {
@@ -42,6 +43,7 @@ class TopicConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConfig
* @return internal topic name set. * @return internal topic name set.
*/ */
def getInternalTopicNameList(): Set[String] = { def getInternalTopicNameList(): Set[String] = {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
withAdminClientAndCatchError(admin => admin.listTopics(new ListTopicsOptions().listInternal(true)).listings() withAdminClientAndCatchError(admin => admin.listTopics(new ListTopicsOptions().listInternal(true)).listings()
.get(timeoutMs, TimeUnit.MILLISECONDS).asScala.filter(_.isInternal).map(_.name()).toSet[String].asJava, .get(timeoutMs, TimeUnit.MILLISECONDS).asScala.filter(_.isInternal).map(_.name()).toSet[String].asJava,
e => { e => {
@@ -69,6 +71,7 @@ class TopicConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConfig
*/ */
def deleteTopic(topic: String): (Boolean, String) = { def deleteTopic(topic: String): (Boolean, String) = {
withAdminClientAndCatchError(admin => { withAdminClientAndCatchError(admin => {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
admin.deleteTopics(Collections.singleton(topic), new DeleteTopicsOptions().retryOnQuotaViolation(false)).all().get(timeoutMs, TimeUnit.MILLISECONDS) admin.deleteTopics(Collections.singleton(topic), new DeleteTopicsOptions().retryOnQuotaViolation(false)).all().get(timeoutMs, TimeUnit.MILLISECONDS)
(true, "") (true, "")
}, },
@@ -103,6 +106,7 @@ class TopicConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConfig
*/ */
def createTopic(topic: NewTopic): (Boolean, String) = { def createTopic(topic: NewTopic): (Boolean, String) = {
withAdminClientAndCatchError(admin => { withAdminClientAndCatchError(admin => {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val createResult = admin.createTopics(Collections.singleton(topic), new CreateTopicsOptions().retryOnQuotaViolation(false)) val createResult = admin.createTopics(Collections.singleton(topic), new CreateTopicsOptions().retryOnQuotaViolation(false))
createResult.all().get(timeoutMs, TimeUnit.MILLISECONDS) createResult.all().get(timeoutMs, TimeUnit.MILLISECONDS)
(true, "") (true, "")
@@ -117,6 +121,7 @@ class TopicConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConfig
*/ */
def createPartitions(newPartitions: util.Map[String, NewPartitions]): (Boolean, String) = { def createPartitions(newPartitions: util.Map[String, NewPartitions]): (Boolean, String) = {
withAdminClientAndCatchError(admin => { withAdminClientAndCatchError(admin => {
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
admin.createPartitions(newPartitions, admin.createPartitions(newPartitions,
new CreatePartitionsOptions().retryOnQuotaViolation(false)).all().get(timeoutMs, TimeUnit.MILLISECONDS) new CreatePartitionsOptions().retryOnQuotaViolation(false)).all().get(timeoutMs, TimeUnit.MILLISECONDS)
(true, "") (true, "")
@@ -241,6 +246,7 @@ class TopicConsole(config: KafkaConfig) extends KafkaConsole(config: KafkaConfig
.asScala.map(info => new TopicPartition(topic, info.partition())).toSeq .asScala.map(info => new TopicPartition(topic, info.partition())).toSeq
case None => throw new IllegalArgumentException("topic is not exist.") case None => throw new IllegalArgumentException("topic is not exist.")
} }
val timeoutMs = ContextConfigHolder.CONTEXT_CONFIG.get().getRequestTimeoutMs()
val offsetMap = KafkaConsole.getLogTimestampOffsets(admin, partitions, timestamp, timeoutMs) val offsetMap = KafkaConsole.getLogTimestampOffsets(admin, partitions, timestamp, timeoutMs)
offsetMap.map(tuple2 => (tuple2._1, tuple2._2.offset())).toMap.asJava offsetMap.map(tuple2 => (tuple2._1, tuple2._2.offset())).toMap.asJava
}, e => { }, e => {

View File

@@ -11,8 +11,8 @@
><router-link to="/group-page" class="pad-l-r">消费组</router-link> ><router-link to="/group-page" class="pad-l-r">消费组</router-link>
<span>|</span <span>|</span
><router-link to="/message-page" class="pad-l-r">消息</router-link> ><router-link to="/message-page" class="pad-l-r">消息</router-link>
<span v-show="config.enableAcl">|</span <span v-show="enableSasl">|</span
><router-link to="/acl-page" class="pad-l-r" v-show="config.enableAcl" ><router-link to="/acl-page" class="pad-l-r" v-show="enableSasl"
>Acl</router-link >Acl</router-link
> >
<span>|</span <span>|</span
@@ -23,7 +23,7 @@
</div> </div>
</template> </template>
<script> <script>
import { KafkaConfigApi, KafkaClusterApi } from "@/utils/api"; import { KafkaClusterApi } from "@/utils/api";
import request from "@/utils/request"; import request from "@/utils/request";
import { mapMutations, mapState } from "vuex"; import { mapMutations, mapState } from "vuex";
import { getClusterInfo } from "@/utils/local-cache"; import { getClusterInfo } from "@/utils/local-cache";
@@ -37,13 +37,6 @@ export default {
}; };
}, },
created() { created() {
request({
url: KafkaConfigApi.getConfig.url,
method: KafkaConfigApi.getConfig.method,
}).then((res) => {
this.config = res.data;
});
const clusterInfo = getClusterInfo(); const clusterInfo = getClusterInfo();
if (!clusterInfo) { if (!clusterInfo) {
request({ request({
@@ -66,6 +59,7 @@ export default {
computed: { computed: {
...mapState({ ...mapState({
clusterName: (state) => state.clusterInfo.clusterName, clusterName: (state) => state.clusterInfo.clusterName,
enableSasl: (state) => state.clusterInfo.enableSasl,
}), }),
}, },
methods: { methods: {

View File

@@ -10,12 +10,22 @@ export default new Vuex.Store({
clusterInfo: { clusterInfo: {
id: undefined, id: undefined,
clusterName: undefined, clusterName: undefined,
enableSasl: false,
}, },
}, },
mutations: { mutations: {
[CLUSTER.SWITCH](state, clusterInfo) { [CLUSTER.SWITCH](state, clusterInfo) {
state.clusterInfo.id = clusterInfo.id; state.clusterInfo.id = clusterInfo.id;
state.clusterInfo.clusterName = clusterInfo.clusterName; state.clusterInfo.clusterName = clusterInfo.clusterName;
let enableSasl = false;
for (let p in clusterInfo.properties) {
if (enableSasl) {
break;
}
enableSasl =
clusterInfo.properties[p].indexOf("security.protocol=SASL") != -1;
}
state.clusterInfo.enableSasl = enableSasl;
setClusterInfo(clusterInfo); setClusterInfo(clusterInfo);
}, },
}, },

View File

@@ -51,7 +51,7 @@ export const KafkaAclApi = {
export const KafkaConfigApi = { export const KafkaConfigApi = {
getConfig: { getConfig: {
url: "/config", url: "/config/console",
method: "get", method: "get",
}, },
getTopicConfig: { getTopicConfig: {

View File

@@ -232,11 +232,13 @@ export default {
} }
}, },
onDeleteUser(row) { onDeleteUser(row) {
this.loading = true;
request({ request({
url: KafkaAclApi.deleteKafkaUser.url, url: KafkaAclApi.deleteKafkaUser.url,
method: KafkaAclApi.deleteKafkaUser.method, method: KafkaAclApi.deleteKafkaUser.method,
data: { username: row.username }, data: { username: row.username },
}).then((res) => { }).then((res) => {
this.loading = false;
this.getAclList(); this.getAclList();
if (res.code == 0) { if (res.code == 0) {
this.$message.success(res.msg); this.$message.success(res.msg);

View File

@@ -8,20 +8,22 @@
:footer="null" :footer="null"
@cancel="handleCancel" @cancel="handleCancel"
> >
<a-form :form="form" :label-col="{ span: 5 }" :wrapper-col="{ span: 12 }"> <a-spin :spinning="loading">
<a-form-item label="用户名"> <a-form :form="form" :label-col="{ span: 5 }" :wrapper-col="{ span: 12 }">
<span>{{ user.username }}</span> <a-form-item label="用户名">
</a-form-item> <span>{{ user.username }}</span>
<a-form-item label="密码"> </a-form-item>
<span>{{ user.password }}</span> <a-form-item label="密码">
</a-form-item> <span>{{ user.password }}</span>
<a-form-item label="凭证信息"> </a-form-item>
<span>{{ user.credentialInfos }}</span> <a-form-item label="凭证信息">
</a-form-item> <span>{{ user.credentialInfos }}</span>
<a-form-item label="数据一致性说明"> </a-form-item>
<strong>{{ user.consistencyDescription }}</strong> <a-form-item label="数据一致性说明">
</a-form-item> <strong>{{ user.consistencyDescription }}</strong>
</a-form> </a-form-item>
</a-form>
</a-spin>
</a-modal> </a-modal>
</template> </template>
@@ -47,6 +49,7 @@ export default {
show: this.visible, show: this.visible,
form: this.$form.createForm(this, { name: "UserDetailForm" }), form: this.$form.createForm(this, { name: "UserDetailForm" }),
user: {}, user: {},
loading: false,
}; };
}, },
watch: { watch: {
@@ -63,11 +66,13 @@ export default {
}, },
getUserDetail() { getUserDetail() {
const api = KafkaAclApi.getKafkaUserDetail; const api = KafkaAclApi.getKafkaUserDetail;
this.loading = true;
request({ request({
url: api.url, url: api.url,
method: api.method, method: api.method,
params: { username: this.username }, params: { username: this.username },
}).then((res) => { }).then((res) => {
this.loading = false;
if (res.code != 0) { if (res.code != 0) {
this.$message.error(res.msg); this.$message.error(res.msg);
} else { } else {

View File

@@ -46,9 +46,9 @@
rows="5" rows="5"
placeholder='可选参数,集群其它属性配置: placeholder='可选参数,集群其它属性配置:
request.timeout.ms=10000 request.timeout.ms=10000
security-protocol=SASL_PLAINTEXT security.protocol=SASL_PLAINTEXT
sasl-mechanism=SCRAM-SHA-256 sasl.mechanism=SCRAM-SHA-256
sasl-jaas-config=org.apache.kafka.common.security.scram.ScramLoginModule required username="name" password="password"; sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="name" password="password";
' '
v-decorator="[ v-decorator="[
'properties', 'properties',

View File

@@ -201,6 +201,7 @@ const columns = [
dataIndex: "properties", dataIndex: "properties",
key: "properties", key: "properties",
scopedSlots: { customRender: "properties" }, scopedSlots: { customRender: "properties" },
width: 300,
}, },
{ {
title: "操作", title: "操作",