最近公司發生了服務定時同步的機制異常，看著公司的分布式任務調度組件，想著自己實現一個 github地址
業務背景
A模塊中的定時器每個30s去執行一次任務(任務內容是遠程獲取消息并處理發送)。原定是A模塊部署到兩個服務器，但是目前A模塊獨立運行兩個之后總是會獲取到相同的遠程消息，然后重復執行處理了兩次。
實際場景
公司采用Spring quartz 建立定時任務模塊。 當任務模塊進行了分布式部署，通常會出現定時任務重復執行的情況。 怎么避免這種情況呢，是否可以構建一個任務注冊中心，Quartz負責注冊任務，但不具體執行任務內的業務邏輯。
target.png
解決問題
任務注冊中心解決任務重復注冊的問題，同時將任務分配給若干處理器進行具體的業務處理，保證在同一個時間內，一個任務只會被一個處理器進行處理。
實現方式
利用ZooKeeper的Master選舉機制實現。 注冊任務就相當于在ZooKeeper中創建或更新一個節點。通過更新節點的內容，來記錄任務的執行狀態。
過程說明
任務調度，從管理器分配任務，根據不同的任務Id進行注冊
public class ZKScheduleManager extends ThreadPoolTaskScheduler implements ApplicationContextAware {
/**
*
*/
private static final long serialVersionUID = 1L;
private static final int DEFAULT_POOL_SIZE = 20;
private static final transient Logger LOGGER = LoggerFactory.getLogger(ZKScheduleManager.class);
private final CountDownLatch downLatch = new CountDownLatch(1);
private Map<String, String> zkConfig;
protected ZKManager zkManager;
private IScheduleDataManager scheduleDataManager;
/**
* 當前調度服務的信息
*/
protected ScheduleServer currenScheduleServer;
/**
* 是否啟動調度管理，如果只是做系統管理，應該設置為false,對應key值為onlyAdmin
*/
public boolean start = true;
/**
* 心跳間隔
*/
private int timerInterval = 1000;
/**
* 是否注冊成功
*/
private boolean isScheduleServerRegister = true;
private static ApplicationContext applicationcontext;
private Map<String, Boolean> isOwnerMap = new ConcurrentHashMap<String, Boolean>();
private Timer hearBeatTimer;
private Lock initLock = new ReentrantLock();
private boolean isStopSchedule = false;
private Lock registerLock = new ReentrantLock();
private List<TaskDefine> initTaskDefines = new ArrayList<TaskDefine>();
private volatile String errorMessage = "No config Zookeeper connect information";
private InitialThread initialThread;
public ZKScheduleManager() {
this.currenScheduleServer = ScheduleServer.createScheduleServer(null);
}
public void init() throws Exception {
if(this.zkConfig != null){
for (Map.Entry<String, String> e : this.zkConfig.entrySet()) {
ConsoleManager.properties.put(e.getKey(), e.getValue());
}
}
if(ConsoleManager.properties.containsKey("onlyClient")){
String val = String.valueOf(ConsoleManager.properties.get("onlyClient"));
if(StringUtils.isNotBlank(val)){
start = Boolean.valueOf(val);
}
}
this.setPoolSize(DEFAULT_POOL_SIZE);
if(ConsoleManager.properties.containsKey("poolSize")){
String val = String.valueOf(ConsoleManager.properties.get("poolSize"));
if(StringUtils.isNotBlank(val)){
this.setPoolSize(Integer.valueOf(val));
}
}
System.out.println("properties:"+ConsoleManager.properties);
this.init(ConsoleManager.properties);
}
public void init(Properties p) throws Exception {
if (this.initialThread != null) {
this.initialThread.stopThread();
}
this.initLock.lock();
try {
this.scheduleDataManager = null;
if (this.zkManager != null) {
this.zkManager.close();
}
this.zkManager = new ZKManager(p);
this.errorMessage = "Zookeeper connecting ......"
+ this.zkManager.getConnectStr();
initialThread = new InitialThread(this);
initialThread.setName("ScheduleManager-initialThread");
initialThread.start();
} finally {
this.initLock.unlock();
}
}
private void rewriteScheduleInfo() throws Exception {
registerLock.lock();
try {
if (this.isStopSchedule) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("外部命令終止調度,不在注冊調度服務，避免遺留垃圾數據："
+ currenScheduleServer.getUuid());
}
return;
}
// 先發送心跳信息
if (errorMessage != null) {
this.currenScheduleServer.setDealInfoDesc(errorMessage);
}
if (!this.scheduleDataManager
.refreshScheduleServer(this.currenScheduleServer)) {
// 更新信息失敗，清除內存數據后重新注冊
this.clearMemoInfo();
this.scheduleDataManager.registerScheduleServer(this.currenScheduleServer);
}
isScheduleServerRegister = true;
} finally {
registerLock.unlock();
}
}
/**
* 清除內存中所有的已經取得的數據和任務隊列,在心態更新失敗，或者發現注冊中心的調度信息被刪除
*/
public void clearMemoInfo() {
try {
} finally {
}
}
/**
* 根據當前調度服務器的信息，重新計算分配所有的調度任務
* 任務的分配是需要加鎖，避免數據分配錯誤。為了避免數據鎖帶來的負面作用，通過版本號來達到鎖的目的
*
* 1、獲取任務狀態的版本號 2、獲取所有的服務器注冊信息和任務隊列信息 3、清除已經超過心跳周期的服務器注冊信息 3、重新計算任務分配
* 4、更新任務狀態的版本號【樂觀鎖】 5、根系任務隊列的分配信息
*
* @throws Exception
*/
public void assignScheduleTask() throws Exception {
scheduleDataManager.clearExpireScheduleServer();
List<String> serverList = scheduleDataManager.loadScheduleServerNames();
if (!scheduleDataManager.isLeader(this.currenScheduleServer.getUuid(),
serverList)) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(this.currenScheduleServer.getUuid()
+ ":不是負責任務分配的Leader,直接返回");
}
return;
}
//黑名單
for(String ip:zkManager.getIpBlacklist()){
int index = serverList.indexOf(ip);
if (index > -1){
serverList.remove(index);
}
}
// 設置初始化成功標準，避免在leader轉換的時候，新增的線程組初始化失敗
scheduleDataManager.assignTask(this.currenScheduleServer.getUuid(), serverList);
}
/**
* 定時向數據配置中心更新當前服務器的心跳信息。 如果發現本次更新的時間如果已經超過了，服務器死亡的心跳周期，則不能在向服務器更新信息。
* 而應該當作新的服務器，進行重新注冊。
*
* @throws Exception
*/
public void refreshScheduleServer() throws Exception {
try {
rewriteScheduleInfo();
// 如果任務信息沒有初始化成功，不做任務相關的處理
if (!this.isScheduleServerRegister) {
return;
}
// 重新分配任務
this.assignScheduleTask();
// 檢查本地任務
this.checkLocalTask();
} catch (Throwable e) {
// 清除內存中所有的已經取得的數據和任務隊列,避免心跳線程失敗時候導致的數據重復
this.clearMemoInfo();
if (e instanceof Exception) {
throw (Exception) e;
} else {
throw new Exception(e.getMessage(), e);
}
}
}
public void checkLocalTask() throws Exception {
// 檢查系統任務執行情況
scheduleDataManager.checkLocalTask(this.currenScheduleServer.getUuid());
}
/**
* 在Zk狀態正常后回調數據初始化
*
* @throws Exception
*/
public void initialData() throws Exception {
this.zkManager.initial();
this.scheduleDataManager = new ScheduleDataManager4ZK(this.zkManager);
checkScheduleDataManager();
if (this.start) {
// 注冊調度管理器
this.scheduleDataManager.registerScheduleServer(this.currenScheduleServer);
if (hearBeatTimer == null) {
hearBeatTimer = new Timer("ScheduleManager-"
+ this.currenScheduleServer.getUuid() + "-HearBeat");
}
hearBeatTimer.schedule(new HeartBeatTimerTask(this), 1000, this.timerInterval);
//初始化啟動數據
if(initTaskDefines != null && initTaskDefines.size() > 0){
for(TaskDefine taskDefine : initTaskDefines){
scheduleDataManager.addTask(taskDefine);
}
}
}
}
private Runnable taskWrapper(final Runnable task){
return new Runnable(){
public void run(){
TaskDefine taskDefine = resolveTaskName(task);
String name = taskDefine.stringKey();
if(StringUtils.isNotEmpty(name)){
boolean isOwner = false;
boolean isRunning = true;
try {
if(!isScheduleServerRegister){
Thread.sleep(1000);
}
if(zkManager.checkZookeeperState()){
isOwner = scheduleDataManager.isOwner(name, currenScheduleServer.getUuid());
isOwnerMap.put(name, isOwner);
isRunning = scheduleDataManager.isRunning(name);
}else{
// 如果zk不可用，使用歷史數據
if(null != isOwnerMap){
isOwner = isOwnerMap.get(name);
}
}
if(isOwner && isRunning){
String msg = null;
try {
task.run();
LOGGER.info("Cron job has been executed.");
} catch (Exception e) {
msg = e.getLocalizedMessage();
}
scheduleDataManager.saveRunningInfo(name, currenScheduleServer.getUuid(), taskDefine.getRunTimes(), msg);
}
} catch (Exception e) {
LOGGER.error("Check task owner error.", e);
}
}
}
};
}
private TaskDefine resolveTaskName(final Runnable task) {
Method targetMethod = null;
TaskDefine taskDefine = new TaskDefine();
if(task instanceof ScheduledMethodRunnable){
ScheduledMethodRunnable runnable = (ScheduledMethodRunnable)task;
taskDefine.setType(TaskDefine.TYPE_UNCODE_SINGLE_TASK);
taskDefine.valueOf(runnable.getTaskDefine());
taskDefine.setRunTimes(runnable.getRunTimes());
}else if(task instanceof ScheduledDistributedSubRunnable){
ScheduledDistributedSubRunnable runnable = (ScheduledDistributedSubRunnable)task;
taskDefine.setType(TaskDefine.TYPE_UNCODE_MULTI_SUB_TASK);
taskDefine.valueOf(runnable.getTaskDefine());
taskDefine.setRunTimes(runnable.getRunTimes());
}else if(task instanceof ScheduledDistributedMainRunnable){
ScheduledDistributedMainRunnable runnable = (ScheduledDistributedMainRunnable)task;
taskDefine.valueOf(runnable.getTaskDefine());
taskDefine.setRunTimes(runnable.getRunTimes());
taskDefine.setType(TaskDefine.TYPE_UNCODE_MULTI_MAIN_TASK);
}else{
org.springframework.scheduling.support.ScheduledMethodRunnable springScheduledMethodRunnable = (org.springframework.scheduling.support.ScheduledMethodRunnable)task;
targetMethod = springScheduledMethodRunnable.getMethod();
taskDefine.setType(TaskDefine.TYPE_SPRING_TASK);
String[] beanNames = applicationcontext.getBeanNamesForType(targetMethod.getDeclaringClass());
if(null != beanNames && StringUtils.isNotEmpty(beanNames[0])){
taskDefine.setTargetBean(beanNames[0]);
taskDefine.setTargetMethod(targetMethod.getName());
}
}
return taskDefine;
}
class HeartBeatTimerTask extends java.util.TimerTask {
private transient final Logger log = LoggerFactory.getLogger(HeartBeatTimerTask.class);
ZKScheduleManager manager;
public HeartBeatTimerTask(ZKScheduleManager aManager) {
manager = aManager;
}
public void run() {
try {
Thread.currentThread().setPriority(Thread.MAX_PRIORITY);
manager.refreshScheduleServer();
} catch (Exception ex) {
log.error(ex.getMessage(), ex);
}
}
}
class InitialThread extends Thread {
private transient Logger log = LoggerFactory.getLogger(InitialThread.class);
ZKScheduleManager sm;
public InitialThread(ZKScheduleManager sm) {
this.sm = sm;
}
boolean isStop = false;
public void stopThread() {
this.isStop = true;
}
@Override
public void run() {
sm.initLock.lock();
try {
int count = 0;
while (!sm.zkManager.checkZookeeperState()) {
count = count + 1;
if (count % 50 == 0) {
sm.errorMessage = "Zookeeper connecting ......"
+ sm.zkManager.getConnectStr() + " spendTime:"
+ count * 20 + "(ms)";
log.error(sm.errorMessage);
}
Thread.sleep(20);
if (this.isStop) {
return;
}
}
sm.initialData();
} catch (Throwable e) {
log.error(e.getMessage(), e);
} finally {
sm.initLock.unlock();
}
}
}
public IScheduleDataManager getScheduleDataManager() {
return scheduleDataManager;
}
@Override
public void setApplicationContext(ApplicationContext applicationcontext)
throws BeansException {
ZKScheduleManager.applicationcontext = applicationcontext;
}
public void setZkManager(ZKManager zkManager) {
this.zkManager = zkManager;
}
public ZKManager getZkManager() {
return zkManager;
}
public void setZkConfig(Map<String, String> zkConfig) {
this.zkConfig = zkConfig;
}
/**
* 使用fixedRate的方式提交任務調度請求
* <pre>
* 任務首次啟動時間未設置，任務池將會盡可能早的啟動任務
* </pre>
*
* @param task 待執行的任務
* @param period 兩次任務啟動時間之間的間隔時間，默認單位是毫秒
* @return 任務句柄
*/
public ScheduledFuture<?> scheduleAtFixedRate(Runnable task, long period) {
try {
TaskDefine taskDefine = resolveTaskName(task);
taskDefine.setPeriod(period);
checkScheduleDataManager();
boolean rt = isUncodeTask(task);
if(rt == false){
scheduleDataManager.addTask(taskDefine);
}
LOGGER.debug(currenScheduleServer.getUuid() +":自動向集群注冊任務[" + taskDefine.stringKey() + "]");
} catch (Exception e) {
LOGGER.error("update task error", e);
}
return super.scheduleAtFixedRate(taskWrapper(task), period);
}
/**
* 提交任務調度請求
*
* @param task 待執行任務
* @param trigger 使用Trigger指定任務調度規則
* @return 任務句柄
*/
public ScheduledFuture<?> schedule(Runnable task, Trigger trigger) {
try {
TaskDefine taskDefine = resolveTaskName(task);
String cronEx = trigger.toString();
int index = cronEx.indexOf(":");
if(index >= 0){
cronEx = cronEx.substring(index + 1);
taskDefine.setCronExpression(cronEx.trim());
}
checkScheduleDataManager();
boolean rt = isUncodeTask(task);
if(rt == false){
scheduleDataManager.addTask(taskDefine);
}
LOGGER.debug(currenScheduleServer.getUuid() +":自動向集群注冊任務[" + taskDefine.getSingalKey() + "]");
} catch (Exception e) {
LOGGER.error("update task error", e);
}
return super.schedule(taskWrapper(task), trigger);
}
/**
* 提交任務調度請求
* <pre>
* 注意任務只執行一次，使用startTime指定其啟動時間
* </pre>
* @param task 待執行任務
* @param startTime 任務啟動時間
* @return 任務句柄
*/
public ScheduledFuture<?> schedule(Runnable task, Date startTime) {
try {
TaskDefine taskDefine = resolveTaskName(task);
taskDefine.setStartTime(startTime);
checkScheduleDataManager();
boolean rt = isUncodeTask(task);
if(rt == false){
scheduleDataManager.addTask(taskDefine);
}
LOGGER.debug(currenScheduleServer.getUuid() +":自動向集群注冊任務[" + taskDefine.getSingalKey() + "]");
} catch (Exception e) {
LOGGER.error("update task error", e);
}
return super.schedule(taskWrapper(task), startTime);
}
private void checkScheduleDataManager() throws InterruptedException {
if(scheduleDataManager == null){
downLatch.await(1000, TimeUnit.MILLISECONDS);
}else{
downLatch.countDown();
}
}
private boolean isUncodeTask(Runnable task){
if(task instanceof ScheduledMethodRunnable){
return true;
}else if(task instanceof ScheduledDistributedSubRunnable){
return true;
}else if(task instanceof ScheduledDistributedMainRunnable){
return true;
}
return false;
}
/**
* 使用fixedRate的方式提交任務調度請求
* <pre>
* 任務首次啟動時間由傳入參數指定
* </pre>
* @param task 待執行的任務
* @param startTime 任務啟動時間
* @param period 兩次任務啟動時間之間的間隔時間，默認單位是毫秒
* @return 任務句柄
*/
public ScheduledFuture<?> scheduleAtFixedRate(Runnable task, Date startTime, long period) {
try {
TaskDefine taskDefine = resolveTaskName(task);
taskDefine.setStartTime(startTime);
taskDefine.setPeriod(period);
checkScheduleDataManager();
boolean rt = isUncodeTask(task);
if(rt == false){
scheduleDataManager.addTask(taskDefine);
}
LOGGER.debug(currenScheduleServer.getUuid() +":自動向集群注冊任務[" + taskDefine.getSingalKey() + "]");
} catch (Exception e) {
LOGGER.error("update task error", e);
}
return super.scheduleAtFixedRate(taskWrapper(task), startTime, period);
}
/**
* 使用fixedDelay的方式提交任務調度請求
* <pre>
* 任務首次啟動時間由傳入參數指定
* </pre>
* @param task 待執行任務
* @param startTime 任務啟動時間
* @param delay 上一次任務結束時間與下一次任務開始時間的間隔時間，單位默認是毫秒
* @return 任務句柄
*/
public ScheduledFuture<?> scheduleWithFixedDelay(Runnable task, Date startTime, long delay) {
try {
TaskDefine taskDefine = resolveTaskName(task);
taskDefine.setStartTime(startTime);
taskDefine.setPeriod(delay);
checkScheduleDataManager();
boolean rt = isUncodeTask(task);
if(rt == false){
scheduleDataManager.addTask(taskDefine);
}
LOGGER.debug(currenScheduleServer.getUuid() +":自動向集群注冊任務[" + taskDefine.getSingalKey() + "]");
} catch (Exception e) {
LOGGER.error("update task error", e);
}
return super.scheduleWithFixedDelay(taskWrapper(task), startTime, delay);
}
/**
* 使用fixedDelay的方式提交任務調度請求
* <pre>
* 任務首次啟動時間未設置，任務池將會盡可能早的啟動任務
* </pre>
* @param task 待執行任務
* @param delay 上一次任務結束時間與下一次任務開始時間的間隔時間，單位默認是毫秒
* @return 任務句柄
*/
public ScheduledFuture<?> scheduleWithFixedDelay(Runnable task, long delay) {
try {
TaskDefine taskDefine = resolveTaskName(task);
taskDefine.setPeriod(delay);
checkScheduleDataManager();
boolean rt = isUncodeTask(task);
if(rt == false){
scheduleDataManager.addTask(taskDefine);
}
LOGGER.debug(currenScheduleServer.getUuid() +":自動向集群注冊任務[" + taskDefine.getSingalKey() + "]");
} catch (Exception e) {
LOGGER.error("update task error", e);
}
return super.scheduleWithFixedDelay(taskWrapper(task), delay);
}
public boolean checkAdminUser(String account, String password){
if(StringUtils.isBlank(account) || StringUtils.isBlank(password)){
return false;
}
String name = zkConfig.get(ZKManager.KEYS.userName.key);
String pwd = zkConfig.get(ZKManager.KEYS.password.key);
if(account.equals(name) && password.equals(pwd)){
return true;
}
return false;
}
public String getScheduleServerUUid(){
if(null != currenScheduleServer){
return currenScheduleServer.getUuid();
}
return null;
}
public Map<String, Boolean> getIsOwnerMap() {
return isOwnerMap;
}
public static ApplicationContext getApplicationcontext() {
return ZKScheduleManager.applicationcontext;
}
public void setInitTaskDefines(List<TaskDefine> initTaskDefines) {
this.initTaskDefines = initTaskDefines;
}
public void destroy() {
try {
if (this.initialThread != null) {
this.initialThread.stopThread();
}
if (this.scheduleDataManager != null) {
this.scheduleDataManager.clearExpireScheduleServer();
}
if (this.hearBeatTimer != null) {
this.hearBeatTimer.cancel();
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (this.zkManager != null) {
try {
this.zkManager.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
}
任務注冊
public class ConsoleManager {
private static transient Logger log = LoggerFactory.getLogger(ConsoleManager.class);
// private static Gson GSON = new GsonBuilder().create();
private static ZKScheduleManager scheduleManager;
static Properties properties = new Properties();
public static void setProperties(Properties prop){
properties.putAll(prop);
}
public static ZKScheduleManager getScheduleManager() throws Exception {
if(null == ConsoleManager.scheduleManager){
synchronized(ConsoleManager.class) {
ConsoleManager.scheduleManager = ZKScheduleManager.getApplicationcontext().getBean(ZKScheduleManager.class);
}
}
return ConsoleManager.scheduleManager;
}
/**
* 添加任務
* @param taskDefine 任務定義
*/
public static void addScheduleTask(TaskDefine taskDefine) {
try {
log.info("添加任務："+taskDefine.getSingalKey());
ConsoleManager.getScheduleManager().getScheduleDataManager().addTask(taskDefine);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* 刪除任務
* @param taskDefine 任務定義
*/
public static void delScheduleTask(TaskDefine taskDefine) {
try {
ConsoleManager.scheduleManager.getScheduleDataManager().delTask(taskDefine);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* 不可用
* @param targetBean bean名稱
* @param targetMethod 方法名稱
*/
@Deprecated
public static void delScheduleTask(String targetBean, String targetMethod) {
try {
ConsoleManager.scheduleManager.getScheduleDataManager().delTask(targetBean, targetMethod);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* 修改任務
* @param taskDefine 任務定義
*/
public static void updateScheduleTask(TaskDefine taskDefine) {
try {
ConsoleManager.scheduleManager.getScheduleDataManager().updateTask(taskDefine);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* 查詢所有任務列表
* @return 任務列表
*/
public static List<TaskDefine> queryScheduleTask() {
List<TaskDefine> taskDefines = new ArrayList<TaskDefine>();
try {
List<TaskDefine> tasks = ConsoleManager.getScheduleManager().getScheduleDataManager().selectTask();
taskDefines.addAll(tasks);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return taskDefines;
}
/**
* 任務是否存在
* @param taskDefine 任務定義
* @return 是或否
* @throws Exception 異常
*/
public static boolean isExistsTask(TaskDefine taskDefine) throws Exception{
return ConsoleManager.scheduleManager.getScheduleDataManager().isExistsTask(taskDefine);
}
/**
* 根據標識查詢相關任務
* @param taskDefine 任務定義
* @return 任務信息
* @throws Exception 異常
*/
public static TaskDefine queryScheduleTask(TaskDefine taskDefine) throws Exception{
return ConsoleManager.scheduleManager.getScheduleDataManager().selectTask(taskDefine);
}
/**
* 判斷當前任務是否屬于當前節點
* @param taskDefine 任務定義
* @return 是或否
* @throws Exception 異常
*/
public static boolean isOwner(TaskDefine taskDefine) throws Exception{
return ConsoleManager.scheduleManager.getScheduleDataManager().isOwner(taskDefine.getSingalKey(),
ConsoleManager.getScheduleManager().getScheduleServerUUid());
}
}
