Commit 6431e029 authored by Laurent Martin's avatar Laurent Martin

upgrade solr version to 6.6.0 and zookeeper version to 3.4.10 + add a way to...

upgrade solr version to 6.6.0 and zookeeper version to 3.4.10 + add a way to reset index via launcher + indicate more info on started collections in lanucher solr status command


git-svn-id: svn://svn.forge.objectweb.org/svnroot/weblab/trunk/WebLabApplications/bundle@4567 a803c184-d8de-4a0e-a52b-d3eef88212b6
parent 8a9dda02
......@@ -23,11 +23,16 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.common.SolrException;
import org.ow2.weblab.bundle.conf.WebLabBean;
import org.ow2.weblab.bundle.server.ActiveMQServer;
import org.ow2.weblab.bundle.server.Bus;
......@@ -43,6 +48,7 @@ import org.ow2.weblab.bundle.utils.ProcessUtils.Env;
import org.ow2.weblab.bundle.utils.Utils;
import org.ow2.weblab.bundle.utils.manager.ProcessManager;
import org.ow2.weblab.bundle.utils.monitor.ServerMonitor;
import org.ow2.weblab.components.solr.utils.SolrUtils;
import org.springframework.util.CollectionUtils;
/**
......@@ -172,12 +178,7 @@ public class Launcher {
serverManager.rmi(command, command2, args);
}
} else if (SOLR.equals(command) && (STOP.equals(command2) || START.equals(command2) || RESTART.equals(command2))) {
ArrayList<SolrServer> solrServers = new ArrayList<>();
for (WebLabServer server : configuration.getServers()) {
if (server instanceof SolrServer) {
solrServers.add((SolrServer) server);
}
}
ArrayList<SolrServer> solrServers = getSolrServers();
if (!CollectionUtils.isEmpty(solrServers)) {
if (START.equals(command2)) {
for (SolrServer solrServer : solrServers) {
......@@ -197,12 +198,7 @@ public class Launcher {
}
}
} else if (ZOOKEEPER.equals(command) && (STOP.equals(command2) || START.equals(command2) || RESTART.equals(command2))) {
ArrayList<ZooKeeperServer> zooServers = new ArrayList<>();
for (WebLabServer server : configuration.getServers()) {
if (server instanceof ZooKeeperServer) {
zooServers.add((ZooKeeperServer) server);
}
}
ArrayList<ZooKeeperServer> zooServers = getZooServers();
if (!CollectionUtils.isEmpty(zooServers)) {
if (START.equals(command2)) {
for (ZooKeeperServer zooServer : zooServers) {
......@@ -245,6 +241,28 @@ public class Launcher {
}
private static ArrayList<SolrServer> getSolrServers() {
ArrayList<SolrServer> solrServers = new ArrayList<>();
for (WebLabServer server : configuration.getServers()) {
if (server instanceof SolrServer) {
solrServers.add((SolrServer) server);
}
}
return solrServers;
}
private static ArrayList<ZooKeeperServer> getZooServers() {
ArrayList<ZooKeeperServer> zooServers = new ArrayList<>();
for (WebLabServer server : configuration.getServers()) {
if (server instanceof ZooKeeperServer) {
zooServers.add((ZooKeeperServer) server);
}
}
return zooServers;
}
/**
* Return true if a command is contained in args list
*
......@@ -280,26 +298,87 @@ public class Launcher {
}
// solr index
// TODO Reimplement something here !
// final String solrDataPath = Launcher.configuration.getSolrData();
// if (solrDataPath == null) {
// Launcher.logger.info("No path to Solr index found. Nothing to delete from index.");
// } else {
// final boolean needRestart = Launcher.configuration.getSolrServer().isEnabled() && Launcher.configuration.getSolrServer().isServerFullyStarted(true);
// if (needRestart) {
// Launcher.logger.info("Solr is running. Stopping it to enable reset of index.");
// ServerManager.stop(Launcher.configuration.getSolrServer(), true, Launcher.logger);
// }
//
// final File index = new File(solrDataPath);
// FileUtils.deleteDirectory(index);
// Launcher.logger.info(index.getAbsolutePath() + " has been removed.");
//
// if (needRestart) {
// Launcher.logger.info("Restarting Solr.");
// ServerManager.start(Launcher.configuration.getSolrServer(), Launcher.logger, true);
// }
// }
HashMap<String, HashMap<String,String>>initCollections=null;
CloudSolrClient solrClient=null;
ArrayList<SolrServer> solrServers = getSolrServers();
for (SolrServer server : solrServers) {
initCollections=((SolrServer) server).getInitCollections();
if(!CollectionUtils.isEmpty(initCollections)) {
solrClient=((SolrServer) server).getSolrClient();
break;
}
}
//we found some initial collections solr instances are working with -> reset them
if(!CollectionUtils.isEmpty(initCollections)) {
//is solr correctly running ?
boolean needRestart=false;
for (final Entry<String, HashMap<String, String>> collection : initCollections.entrySet()) {
final String collectionName = collection.getKey();
try {
if(!SolrUtils.checkLeaders(solrClient, collectionName)) {
needRestart=true;
break;
}
}catch(SolrException e) {
Launcher.logger.warning(e.getMessage()+" while checking solr status -> will try to restart it");
needRestart=true;
break;
}
}
ArrayList<ZooKeeperServer> zooServers = getZooServers();
//restart if needed
if(needRestart) {
//stop all solr nodes
for (SolrServer server : solrServers) {
server.stop();
}
//stop all zookeeper nodes
for (ZooKeeperServer server : zooServers) {
server.stop();
}
//start all zookeeper nodes
for (ZooKeeperServer server : zooServers) {
server.start();
}
//start all solr nodes
for (SolrServer server : solrServers) {
server.start();
}
}
//then delete all collections
for (final Entry<String, HashMap<String, String>> collection : initCollections.entrySet()) {
final String collectionName=collection.getKey();
Launcher.logger.info("deleting solr collection : "+collectionName);
SolrUtils.deleteCollection(solrClient, collectionName, 30);
}
//then restart zookeeper instances and solr nodes
//stop all solr nodes
for (SolrServer server : solrServers) {
server.stop();
}
//stop all zookeeper nodes
for (ZooKeeperServer server : zooServers) {
server.stop();
}
//start all zookeeper nodes
for (ZooKeeperServer server : zooServers) {
server.start();
}
//start all solr nodes
for (SolrServer server : solrServers) {
server.start();
}
}
// Content
final String contentFolder = Launcher.configuration.getContentFolder();
......@@ -337,7 +416,7 @@ public class Launcher {
}
}
} catch (final IOException exception) {
} catch (final IOException | SolrServerException | InterruptedException exception) {
Launcher.logger.log(Level.SEVERE, exception.getLocalizedMessage(), exception);
}
}
......
......@@ -24,16 +24,21 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.ow2.weblab.bundle.Constants;
import org.ow2.weblab.bundle.conf.WebLabBean;
import org.ow2.weblab.bundle.server.WebLabServer.State;
import org.ow2.weblab.bundle.utils.PathUtils;
import org.ow2.weblab.bundle.utils.ProcessUtils;
import org.ow2.weblab.bundle.utils.Utils;
import org.ow2.weblab.components.solr.utils.SolrUtils;
import org.springframework.util.CollectionUtils;
/**
* Manage Server related commands
......@@ -92,6 +97,39 @@ public class ServerManager {
logger.info(ServerManager.getServerDisplayedName(applicationServer) + " server status : NOT STARTED.");
}
}
private static final void displayStatus(final SolrServer solrServer, final Logger logger) {
if (!solrServer.isEnabled()) {
if (logger.isLoggable(Level.FINER)) {
logger.finer(ServerManager.getServerDisplayedName(solrServer) + " server is disable");
}
return;
}
if (solrServer.isServerFullyStarted(true)) {
logger.info(ServerManager.getServerDisplayedName(solrServer) + " server status : READY");
//is this solr node the "main" one (the one we used to create inital collections)
HashMap<String, HashMap<String, String>> initCollections=solrServer.getInitCollections();
if(!CollectionUtils.isEmpty(initCollections)) {
final CloudSolrClient solrClient=solrServer.getSolrClient();
for(String collection : initCollections.keySet()) {
logger.info("Collection "+collection+" : READY");
final Map<String, List<String>> replicas=SolrUtils.getReplicas(solrClient, collection);
for(Entry<String, List<String>> entry : replicas.entrySet()) {
logger.info("\tNode "+entry.getKey());
for(String replica : entry.getValue()) {
logger.info("\t\tReplica "+replica+" READY");
}
}
}
}else {
logger.info("No solr collection available");
}
} else {
logger.info(ServerManager.getServerDisplayedName(solrServer) + " server status : NOT STARTED.");
}
}
/**
......@@ -105,7 +143,12 @@ public class ServerManager {
for (final WebLabServer otherServer : this.configuration.getServers()) {
if (server == null || server.equals(otherServer.getName())) {
ServerManager.displayStatus(otherServer, this.logger);
if(otherServer instanceof SolrServer) {
ServerManager.displayStatus((SolrServer)otherServer, this.logger);
}else {
ServerManager.displayStatus(otherServer, this.logger);
}
}
}
......
......@@ -314,6 +314,7 @@ public class SolrServer extends AbstractApplicationServer {
this.logger.log(Level.FINE, "maxShardPerNode property is either not set or not valid.", ignored);
}
SolrUtils.createCollection(this.solrClient, collectionName, configName, shards, routerField, numReplicas, maxShardPerNode, 300);
this.logger.log(Level.INFO, "Creating new solr collection: "+collectionName);
}
}
}
......@@ -394,7 +395,24 @@ public class SolrServer extends AbstractApplicationServer {
this.initCollections = initCollections;
}
/**
* Get list of collections to initialize on instance startup
*
* @return initCollections
* {@link HashMap} list of collections to initalize on instance startup
*/
public HashMap<String, HashMap<String, String>> getInitCollections() {
return this.initCollections;
}
/**
* Get solr client used by this server
* @return {@link CloudSolrClient} solr client used by this server
*/
public CloudSolrClient getSolrClient() {
return solrClient;
}
/**
* @return the dataDirectory
*/
......
......@@ -49,8 +49,8 @@
<tomcat.version>7.0.59</tomcat.version>
<liferay.version>6.1.2-ce-ga3</liferay.version>
<activemq.server.version>5.13.3</activemq.server.version>
<solr.server.version>6.4.0</solr.server.version>
<zookeeper.bin.version>3.4.9</zookeeper.bin.version>
<solr.server.version>6.6.0</solr.server.version>
<zookeeper.bin.version>3.4.10</zookeeper.bin.version>
<!-- Server paths -->
<tomcat.path>apache-tomcat-${tomcat.version}</tomcat.path>
......@@ -82,7 +82,7 @@
<service.simple-gazetteer.version>1.4.2</service.simple-gazetteer.version>
<service.ngramj-language-extraction.version>1.6.2</service.ngramj-language-extraction.version>
<service.gate-extraction.version>2.9.0</service.gate-extraction.version>
<service.solr-engine.version>2.7.0</service.solr-engine.version>
<service.solr-engine.version>2.8.0-SNAPSHOT</service.solr-engine.version>
<!-- Portlets versions -->
<portlet.result-portlet.version>2.1.1</portlet.result-portlet.version>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment