课题_自己封装的一个SolrDataImportRequestHandlerScheduler

自己封装的一个SolrDataImportRequestHandlerScheduler经过将近一天的努力，终于搞定了Solr的 Data Import Request Handler Scheduler。

Scheduler主要解决两个问题：1.定时增量更新索引。

2.定时重做索引。

经过测试，Scheduler已经可以实现完全基于配置，无需开发功能，无需人工干预的情况下实现以上两个功能（结合Solr 的 Data Import Request Handler前提下）。

这里贴出一下主要的代码备忘：BaseTimerTask.java TimerTask基类,封装了一些基本的属性读取、请求发送方法:1package org.apache.solr.handler.dataimport.scheduler;23import java.io.IOException;4import .HttpURLConnection;5import .MalformedURLException;6import .URL;7import java.text.DateFormat;8import java.text.ParseException;9import java.text.SimpleDateFormat;10import java.util.Date;11import java.util.Timer;12import java.util.TimerTask;1314import org.slf4j.Logger;15import org.slf4j.LoggerFactory;1617public abstract class BaseTimerTask extends TimerTask {18protected String syncEnabled;19protected String[] syncCores;20protected String server;21protected String port;22protected String webapp;23protected String params;24protected String interval;25protected String cores;26protected SolrDataImportProperties p;27protected boolean singleCore;2829protected String reBuildIndexParams;30protected String reBuildIndexBeginTime;31protected String reBuildIndexInterval;3233protected static final Logger logger = LoggerFactory34.getLogger(BaseTimerTask.class);3536public BaseTimerTask(String webAppName, Timer t) throws Exception {37// load properties from global dataimport.properties38p = new SolrDataImportProperties();39reloadParams();40fixParams(webAppName);4142if (!syncEnabled.equals("1"))43throw new Exception("Schedule disabled");4445if (syncCores == null46|| (syncCores.length == 1 && syncCores[0].isEmpty())) {47singleCore = true;("<index update process> Single core identified in dataimport.properties");49} else {50singleCore = false;("<index update process> Multiple cores identified in dataimport.properties. Sync active for: " 52+ cores);53}54}5556protected void reloadParams() {57p.loadProperties(true);58syncEnabled = p.getProperty(SolrDataImportProperties.SYNC_ENABLED);59cores = p.getProperty(SolrDataImportProperties.SYNC_CORES);60server = p.getProperty(SolrDataImportProperties.SERVER);61port = p.getProperty(SolrDataImportProperties.PORT);62webapp = p.getProperty(SolrDataImportProperties.WEBAPP);63params = p.getProperty(SolrDataImportProperties.PARAMS);64interval = p.getProperty(SolrDataImportProperties.INTERVAL);65syncCores = cores != null ? cores.split(",") : null;6667reBuildIndexParams = p68.getProperty(SolrDataImportProperties.REBUILDINDEXPARAMS); 69reBuildIndexBeginTime = p70.getProperty(SolrDataImportProperties.REBUILDINDEXBEGINTIME); 71reBuildIndexInterval = p72.getProperty(SolrDataImportProperties.REBUILDINDEXINTERVAL); 7374}7576protected void fixParams(String webAppName) {77if (server == null || server.isEmpty())78server = "localhost";79if (port == null || port.isEmpty())80port = "8080";81if (webapp == null || webapp.isEmpty())82webapp = webAppName;83if (interval == null || interval.isEmpty() || getIntervalInt() <= 0)84interval = "30";85if (reBuildIndexBeginTime == null || reBuildIndexBeginTime.isEmpty())86interval = "00:00:00";87if (reBuildIndexInterval == null || reBuildIndexInterval.isEmpty()88|| getReBuildIndexIntervalInt() <= 0)89reBuildIndexInterval = "0";90}9192protected void prepUrlSendHttpPost(String params) {93String coreUrl = "http://" + server + ":" + port + "/" + webapp94+ params;95sendHttpPost(coreUrl, null);96}9798protected void prepUrlSendHttpPost(String coreName, String params) {99String coreUrl = "http://" + server + ":" + port + "/" + webapp + "/"100+ coreName + params;101sendHttpPost(coreUrl, coreName);102}103104protected void sendHttpPost(String completeUrl, String coreName) {105DateFormat df = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss SSS");106Date startTime = new Date();107108// prepare the core var109String core = coreName == null ? "" : "[" + coreName + "] ";110(core112+ "<index update process> Process started at .............. "113+ df.format(startTime));114115try {116117URL url = new URL(completeUrl);118HttpURLConnection conn = (HttpURLConnection) url.openConnection(); 119120conn.setRequestMethod("POST");121conn.setRequestProperty("type", "submit");122conn.setDoOutput(true);123124// Send HTTP POST125conn.connect();126(core + "<index update process> Full URL\t\t\t\t"128+ conn.getURL());(core + "<index update process> Response message\t\t\t"130+ conn.getResponseMessage());(core + "<index update process> Response code\t\t\t"132+ conn.getResponseCode());133134// listen for change in properties file if an error occurs135if (conn.getResponseCode() != 200) {136reloadParams();137}138139conn.disconnect();(core141+ "<index update process> Disconnected from server\t\t"142+ server);143Date endTime = new Date();(core145+ "<index update process> Process ended at ................ "146+ df.format(endTime));147} catch (MalformedURLException mue) {148logger.error("Failed to assemble URL for HTTP POST", mue);149} catch (IOException ioe) {150logger.error(151"Failed to connect to the specified URL while trying to send HTTP POST", 152ioe);153} catch (Exception e) {154logger.error("Failed to send HTTP POST", e);155}156}157158public int getIntervalInt() {159try {160return Integer.parseInt(interval);161} catch (NumberFormatException e) {162logger.warn(163"Unable to convert 'interval' to number. Using default value (30) instead", 164e);165return 30; // return default in case of error166}167}168169public int getReBuildIndexIntervalInt() {170try {171return Integer.parseInt(reBuildIndexInterval);172} catch (NumberFormatException e) {(174"Unable to convert 'reBuildIndexInterval' to number. do't rebuild index.", 175e);176return 0; // return default in case of error177}178}179180public Date getReBuildIndexBeginTime() {181Date beginDate = null;182try {183SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd");184String dateStr = sdfDate.format(new Date());185beginDate = sdfDate.parse(dateStr);186if (reBuildIndexBeginTime == null187|| reBuildIndexBeginTime.isEmpty()) {189}190if (reBuildIndexBeginTime.matches("\\d{2}:\\d{2}:\\d{2}")) {191SimpleDateFormat sdf = new SimpleDateFormat(192"yyyy-MM-dd HH:mm:ss");193beginDate = sdf.parse(dateStr + " " + reBuildIndexBeginTime);194} else if (reBuildIndexBeginTime195.matches("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}")) {196SimpleDateFormat sdf = new SimpleDateFormat(197"yyyy-MM-dd HH:mm:ss");198beginDate = sdf.parse(reBuildIndexBeginTime);199}200return beginDate;201} catch (ParseException e) {202logger.warn(203"Unable to convert 'reBuildIndexBeginTime' to date. use now time.",204e);205return beginDate;206}207}208209 }DeltaImportHTTPPostScheduler.java 增量索引更新任务计划:1package org.apache.solr.handler.dataimport.scheduler;23import java.util.Timer;45import org.slf4j.Logger;6import org.slf4j.LoggerFactory;78/**9* 增量更新索引的任务10* @author zhangliang11*12*/13public class DeltaImportHTTPPostScheduler extends BaseTimerTask {1415private static final Logger logger = LoggerFactory16.getLogger(DeltaImportHTTPPostScheduler.class);1718public DeltaImportHTTPPostScheduler(String webAppName, Timer t)19throws Exception {20super(webAppName, t);("<index update process> DeltaImportHTTPPostScheduler init");22}2324public void run() {25try {26// check mandatory params27if (server.isEmpty() || webapp.isEmpty() || params == null28|| params.isEmpty()) {29logger.warn("<index update process> Insuficient info provided for data import"); ("<index update process> Reloading global dataimport.properties"); 31reloadParams();32// single-core33} else if (singleCore) {34prepUrlSendHttpPost(params);3536// multi-core37} else if (syncCores.length == 038|| (syncCores.length == 1 && syncCores[0].isEmpty())) {39logger.warn("<index update process> No cores scheduled for data import");("<index update process> Reloading global dataimport.properties");4243} else {44for (String core : syncCores) {45prepUrlSendHttpPost(core, params);46}47}48} catch (Exception e) {49logger.error("Failed to prepare for sendHttpPost", e);50reloadParams();51}52}53 }FullImportHTTPPostScheduler.java 重做索引任务计划:1package org.apache.solr.handler.dataimport.scheduler;23import java.util.Timer;45import org.slf4j.Logger;6import org.slf4j.LoggerFactory;78/**9* 重做索引的任务10* @author zhangliang11*12*/13public class FullImportHTTPPostScheduler extends BaseTimerTask {1415private static final Logger logger = LoggerFactory16.getLogger(FullImportHTTPPostScheduler.class);1718public FullImportHTTPPostScheduler(String webAppName, Timer t)19throws Exception {20super(webAppName, t);("<index update process> DeltaImportHTTPPostScheduler init");22}2324public void run() {25try {26// check mandatory params27if (server.isEmpty() || webapp.isEmpty()28|| reBuildIndexParams == null29|| reBuildIndexParams.isEmpty()) {30logger.warn("<index update process> Insuficient info provided for data import, reBuildIndexParams is null"); ("<index update process> Reloading global dataimport.properties");32reloadParams();33// single-core34} else if (singleCore) {35prepUrlSendHttpPost(reBuildIndexParams);3637// multi-core38} else if (syncCores.length == 039|| (syncCores.length == 1 && syncCores[0].isEmpty())) {40logger.warn("<index update process> No cores scheduled for data import");("<index update process> Reloading global dataimport.properties");42reloadParams();4344} else {45for (String core : syncCores) {46prepUrlSendHttpPost(core, reBuildIndexParams);47}48}49} catch (Exception e) {50logger.error("Failed to prepare for sendHttpPost", e);51reloadParams();52}53}54 }ApplicationListener.java 调用任务计划的Listener:1package org.apache.solr.handler.dataimport.scheduler;23import java.util.Calendar;4import java.util.Date;5import java.util.Timer;67import javax.servlet.ServletContext;8import javax.servlet.ServletContextEvent;9import javax.servlet.ServletContextListener;1011import org.slf4j.Logger;12import org.slf4j.LoggerFactory;1314public class ApplicationListener implements ServletContextListener {1516private static final Logger logger = LoggerFactory17.getLogger(ApplicationListener.class);1819@Override20public void contextDestroyed(ServletContextEvent servletContextEvent) {21ServletContext servletContext = servletContextEvent.getServletContext();2223// get our timer from the context24Timer timer = (Timer) servletContext.getAttribute("timer");25Timer fullImportTimer = (Timer) servletContext26.getAttribute("fullImportTimer");2728// cancel all active tasks in the timers queue29if (timer != null)30timer.cancel();31if (fullImportTimer != null)32fullImportTimer.cancel();3334// remove the timer from the context35servletContext.removeAttribute("timer");36servletContext.removeAttribute("fullImportTimer");3738}3940@Override41public void contextInitialized(ServletContextEvent servletContextEvent) {42ServletContext servletContext = servletContextEvent.getServletContext();43try {44// 增量更新任务计划45// create the timer and timer task objects46Timer timer = new Timer();47DeltaImportHTTPPostScheduler task = new DeltaImportHTTPPostScheduler( 48servletContext.getServletContextName(), timer);4950// get our interval from HTTPPostScheduler51int interval = task.getIntervalInt();5253// get a calendar to set the start time (first run)54Calendar calendar = Calendar.getInstance();5556// set the first run to now + interval (to avoid fireing while the57// app/server is starting)58calendar.add(Calendar.MINUTE, interval);59Date startTime = calendar.getTime();6061// schedule the task62timer.scheduleAtFixedRate(task, startTime, 1000 * 60 * interval);6364// save the timer in context65servletContext.setAttribute("timer", timer);6667// 重做索引任务计划68Timer fullImportTimer = new Timer();69FullImportHTTPPostScheduler fullImportTask = new FullImportHTTPPostScheduler(70servletContext.getServletContextName(), fullImportTimer);7172int reBuildIndexInterval = fullImportTask73.getReBuildIndexIntervalInt();74if (reBuildIndexInterval <= 0) {75logger.warn("Full Import Schedule disabled");76return;77}7879Calendar fullImportCalendar = Calendar.getInstance();80Date beginDate = fullImportTask.getReBuildIndexBeginTime();81fullImportCalendar.setTime(beginDate);82fullImportCalendar.add(Calendar.MINUTE, reBuildIndexInterval);83Date fullImportStartTime = fullImportCalendar.getTime();8485// schedule the task86fullImportTimer.scheduleAtFixedRate(fullImportTask,87fullImportStartTime, 1000 * 60 * reBuildIndexInterval);8889// save the timer in context90servletContext.setAttribute("fullImportTimer", fullImportTimer);9192} catch (Exception e) {93if (e.getMessage().endsWith("disabled")) {94logger.warn("Schedule disabled");95} else {96logger.error("Problem initializing the scheduled task: ", e);97}98}99100}101102 }使用说明1.将上面的编译文件打包成apache-solr-dataimportscheduler-1.0.jar，然后和solr自带的apache-solr-dataimporthandler-*.jar, apache-solr-dataimporthandler-extras-*.jar 放到solr.war的lib目录下面2.修改solr.war中WEB-INF/web.xml, 在servlet节点前面增加:<listener><listener-class>org.apache.solr.handler.dataimport.scheduler.ApplicationListener</listener-class></listener>3.将apache-solr-dataimportscheduler-.jar 中dataimport.properties 取出并根据实际情况修改,然后放到solr.home/conf (不是solr.home/core/conf) 目录下面4.重启tomcat或者jboss 即可dataimport.properties 配置项说明################################################## ## dataimport scheduler properties ## ################################################### to sync or not to sync# 1 - active; anything else - inactivesyncEnabled=1# which cores to schedule# in a multi-core environment you can decide which cores you want syncronized# leave empty or comment it out if using single-core deploymentsyncCores=core1,core2# solr server name or IP address# [defaults to localhost if empty]server=localhost# solr server port# [defaults to 80 if empty]port=8080# application name/context# [defaults to current ServletContextListener's context (app) name]webapp=solr# URL params [mandatory]# remainder of URLparams=/dataimport?command=delta-import&clean=false&commit=true# schedule interval# number of minutes between two runs# [defaults to 30 if empty]interval=1# 重做索引的时间间隔，单位分钟，默认7200，即5天;# 为空,为0,或者注释掉:表示永不重做索引reBuildIndexInterval=7200# 重做索引的参数reBuildIndexParams=/dataimport?command=full-import&clean=true&commit=true# 重做索引时间间隔的计时开始时间，第一次真正执行的时间=reBuildIndexBeginTime+reBuildIndexInterval*60*1000；# 两种格式：2012-04-11 03:10:00 或者03:10:00，后一种会自动补全日期部分为服务启动时的日期reBuildIndexBeginTime=03:10:00。