Commit 7bc13340 authored by Arnaud Saval's avatar Arnaud Saval
Browse files

WEBLAB-812 launcher will stop when starting only petals

WEBLAB-827, WEBLAB-514, WEBLAB-525 add an optional registry file to manage providers/consumers and chains

Upgrade WebLab Bundle to version 1.2.5.1-SNAPSHOT

git-svn-id: svn://svn.forge.objectweb.org/svnroot/weblab/trunk/WebLabApplications/bundle@3205 a803c184-d8de-4a0e-a52b-d3eef88212b6
parent 993281cf
......@@ -2,7 +2,7 @@
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>1.2.5</id>
<id>1.2.5.1-SNAPSHOT</id>
<formats>
<format>zip</format>
</formats>
......@@ -85,9 +85,14 @@
<source>conf/contentManager.properties</source>
<outputDirectory>conf</outputDirectory>
</file>
<file>
<!-- Service registry -->
<source>conf/registry.xml</source>
<outputDirectory>conf</outputDirectory>
</file>
<file>
<!-- Copy WebLab Launcher -->
<source>libraries/weblab-launcher/target/weblab-launcher-1.0.0-jar-with-dependencies.jar</source>
<source>libraries/weblab-launcher/target/weblab-launcher-1.0.1-SNAPSHOT-jar-with-dependencies.jar</source>
<destName>weblab-launcher.jar</destName>
</file>
<file>
......
......@@ -16,50 +16,6 @@
<!-- Copy su in sa install dir -->
<source>su-SOAP-ChainService-consume/target/su-SOAP-ChainService-consume.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-SolrSearcherService-provide/target/su-SOAP-SolrSearcherService-provide.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-SolrSearcherService-consume/target/su-SOAP-SolrSearcherService-consume.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-DefaultFolderListenerService-provide/target/su-SOAP-DefaultFolderListenerService-provide.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-WarcListenerService-provide/target/su-SOAP-WarcListenerService-provide.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-SolrIndexerService-provide/target/su-SOAP-SolrIndexerService-provide.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-SimpleRepoService-provide/target/su-SOAP-SimpleRepoService-provide.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-SimpleRepoService-consume/target/su-SOAP-SimpleRepoService-consume.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-TikaExtractorServiceService-provide/target/su-SOAP-TikaExtractorServiceService-provide.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-GazetteerImplService-provide/target/su-SOAP-GazetteerImplService-provide.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-LanguageExtractionService-provide/target/su-SOAP-LanguageExtractionService-provide.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-SOAP-GateServiceService-provide/target/su-SOAP-GateServiceService-provide.zip</source>
</file>
<file>
<!-- Copy su in sa install dir -->
<source>su-CHAIN-provide/target/su-CHAIN-provide.zip</source>
......
......@@ -5,116 +5,6 @@
<name>sa-weblab-services</name>
<description></description>
</identification>
<service-unit>
<identification>
<name>su-SOAP-DefaultFolderListenerService-provide</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-DefaultFolderListenerService-provide.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-GateServiceService-provide</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-GateServiceService-provide.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-GazetteerImplService-provide</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-GazetteerImplService-provide.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-LanguageExtractionService-provide</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-LanguageExtractionService-provide.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-WarcListenerService-provide</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-WarcListenerService-provide.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-SolrIndexerService-provide</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-SolrIndexerService-provide.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-SolrSearcherService-consume</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-SolrSearcherService-consume.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-SolrSearcherService-provide</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-SolrSearcherService-provide.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-SimpleRepoService-provide</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-SimpleRepoService-provide.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-SimpleRepoService-consume</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-SimpleRepoService-consume.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-SOAP-TikaExtractorServiceService-provide</name>
<description></description>
</identification>
<target>
<artifacts-zip>su-SOAP-TikaExtractorServiceService-provide.zip</artifacts-zip>
<component-name>petals-bc-soap</component-name>
</target>
</service-unit>
<service-unit>
<identification>
<name>su-POJO-ChainService-provide</name>
......
......@@ -5,7 +5,7 @@
<parent>
<groupId>org.ow2.weblab.bundle</groupId>
<artifactId>versions</artifactId>
<version>1.2.5</version>
<version>1.2.5.1-SNAPSHOT</version>
<relativePath>../versions/pom.xml</relativePath>
</parent>
......@@ -15,21 +15,9 @@
<packaging>pom</packaging>
<modules>
<module>su-SOAP-DefaultFolderListenerService-provide</module>
<module>su-SOAP-WarcListenerService-provide</module>
<module>su-SOAP-TikaExtractorServiceService-provide</module>
<module>su-SOAP-GazetteerImplService-provide</module>
<module>su-SOAP-LanguageExtractionService-provide</module>
<module>su-SOAP-GateServiceService-provide</module>
<module>su-SOAP-SolrIndexerService-provide</module>
<module>su-SOAP-SimpleRepoService-provide</module>
<module>su-POJO-ChainService-provide</module>
<module>su-CHAIN-provide</module>
<module>su-SOAP-ChainService-consume</module>
<module>su-SOAP-SolrSearcherService-provide</module>
<module>su-SOAP-SolrSearcherService-consume</module>
<module>su-SOAP-SimpleRepoService-consume</module>
<!-- <module>sa-weblab-services</module> -->
</modules>
<build>
......
......@@ -7,7 +7,7 @@
<parent>
<groupId>org.ow2.weblab.bundle</groupId>
<artifactId>versions</artifactId>
<version>1.2.5</version>
<version>1.2.5.1-SNAPSHOT</version>
<relativePath>../../versions/pom.xml</relativePath>
</parent>
......
......@@ -7,7 +7,7 @@
<parent>
<groupId>org.ow2.weblab.bundle</groupId>
<artifactId>versions</artifactId>
<version>1.2.5</version>
<version>1.2.5.1-SNAPSHOT</version>
<relativePath>../../versions/pom.xml</relativePath>
</parent>
......@@ -65,6 +65,11 @@
<artifactId>resources-merger</artifactId>
<version>1.1.0</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
......
/**
* hop
* WEBLAB: Service oriented integration platform for media mining and intelligence applications
*
* Copyright (C) 2004 - 2013 CASSIDIAN
*
* This library is free software; you can redistribute it and/or modify it under the terms of
* the GNU Lesser General Public License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License along with this
* library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth
* Floor, Boston, MA 02110-1301 USA
*/
package org.ow2.weblab;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
......@@ -14,37 +27,17 @@ import javax.jbi.messaging.MessagingException;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.namespace.QName;
import javax.xml.transform.dom.DOMResult;
import org.ow2.petals.component.framework.api.exception.PEtALSCDKException;
import org.ow2.petals.component.framework.api.message.Exchange;
import org.ow2.weblab.bundle.server.Chain;
import org.ow2.weblab.chain.ChainRequestType;
import org.ow2.weblab.chain.ChainResponse;
import org.ow2.weblab.chain.ObjectFactory;
import org.ow2.weblab.components.ResourcesMerger;
import org.ow2.weblab.core.model.Resource;
import org.ow2.weblab.core.model.processing.WProcessingAnnotator;
import org.ow2.weblab.core.services.AccessDeniedException;
import org.ow2.weblab.core.services.Analyser;
import org.ow2.weblab.core.services.ContentNotAvailableException;
import org.ow2.weblab.core.services.Indexer;
import org.ow2.weblab.core.services.InsufficientResourcesException;
import org.ow2.weblab.core.services.InvalidParameterException;
import org.ow2.weblab.core.services.QueueManager;
import org.ow2.weblab.core.services.ResourceContainer;
import org.ow2.weblab.core.services.ServiceNotConfiguredException;
import org.ow2.weblab.core.services.UnexpectedException;
import org.ow2.weblab.core.services.UnsupportedRequestException;
import org.ow2.weblab.core.services.analyser.ProcessArgs;
import org.ow2.weblab.core.services.analyser.ProcessReturn;
import org.ow2.weblab.core.services.indexer.IndexArgs;
import org.ow2.weblab.core.services.queuemanager.NextResourceArgs;
import org.ow2.weblab.core.services.resourcecontainer.SaveResourceArgs;
import org.ow2.weblab.petals.JbiServiceFactory;
import org.ow2.weblab.rdf.Value;
import org.ow2.weblab.utils.WebLabRunnable;
import org.purl.dc.elements.DublinCoreAnnotator;
import org.ow2.weblab.core.services.EmptyQueueException;
import org.springframework.beans.factory.xml.XmlBeanFactory;
import org.springframework.core.io.UrlResource;
import org.w3c.dom.Document;
/**
......@@ -55,56 +48,28 @@ import org.w3c.dom.Document;
*/
public class ProcessingChain {
/**
* Context to process files (default context)
*/
public static final String FILES_CONTEXT = "files";
/**
* Context to process warcs
*/
public static final String WARCS_CONTEXT = "warcs";
public static final String CHAIN_REGISTRY = "weblab.chains.registry";
private Logger logger;
private ComponentContext ctx;
/*
* services used in the processing chain instantiate in initServices method
*/
private QueueManager folderQueueManager;
private QueueManager warcQueueManager;
private Analyser tika;
private Analyser simpleGazetteer;
private Analyser ngramj;
private Analyser gate;
private Indexer solr;
private ResourceContainer repository;
private HashMap<String, Chain> chainManager;
public void setComponentContext(ComponentContext ctx) throws JAXBException {
this.logger.info("Setting component ctx: " + ctx);
this.ctx = ctx;
initServices();
initChains();
}
......@@ -114,7 +79,7 @@ public class ProcessingChain {
public boolean onExchange(Exchange exchange) throws MessagingException, PEtALSCDKException, InterruptedException, JAXBException {
initServices();
initChains();
JAXBContext jaxbContext = JAXBContext.newInstance(ObjectFactory.class);
ObjectFactory objectFactory = new ObjectFactory();
......@@ -123,53 +88,28 @@ public class ProcessingChain {
ChainRequestType.class)).getValue();
ChainResponse responseBean = new ChainResponse();
/*
* initialize folderCrawlerRequest with a usageContext (only
* mandatory for the folderCrawler)
*/
NextResourceArgs nextResourceArgs = new NextResourceArgs();
nextResourceArgs.setUsageContext("urn:tutorial");
String usageContext = request.getUsageContext();
// retrieve chain from manager depending on the usage context
Chain chain = chainManager.get(request.getUsageContext());
if (chain == null){
this.logger.severe("No Chain found for usageContext : "+usageContext+". Defined usageContext are : "+chainManager.keySet());
throw new MessagingException("No chain to execute with usageContext "+usageContext);
}
try {
Resource currentResource = null;
/*
* select queuemanager depending on the context
*/
QueueManager queueManager = this.folderQueueManager;
if (request.getUsageContext() != null && ProcessingChain.WARCS_CONTEXT.equals(request.getUsageContext())) {
queueManager = this.warcQueueManager;
}
logger.info("Selected Queue Manager is " + queueManager.getClass().getName() + ", context is " + request.getUsageContext());
/*
* call the folderCrawler and get the next resource to process
*/
currentResource = queueManager.nextResource(nextResourceArgs).getResource();
try {
// Run the processing chain on the resource
processResource(currentResource);
responseBean.setNbDocOK(1);
} catch (Exception e) {
this.logger.warning("Error during processing resource: " + e.getMessage());
this.logger.warning("Continuing with next resource");
responseBean.setNbDocError(1);
}
this.logger.info("Nb doc ok " + responseBean.getNbDocOK() + " - Nb doc error " + responseBean.getNbDocError());
chain.run(logger, usageContext);
responseBean.setNbDocOK(1);
} catch (EmptyQueueException e) {
this.logger.warning("Queue is empty: " + e.getMessage());
logger.log(Level.WARNING,e.getMessage(),e);
} catch (Exception e) {
/*
* something wrong during service calling
*/
this.logger.warning("Error during chain calling: " + e.getMessage());
e.printStackTrace();
this.logger.warning("Error during processing resource: " + e.getMessage());
this.logger.warning("Continuing with next resource");
responseBean.setNbDocError(1);
}
this.logger.info("Nb doc ok " + responseBean.getNbDocOK() + " - Nb doc error " + responseBean.getNbDocError());
/*
* reply using response bean
*/
......@@ -180,169 +120,30 @@ public class ProcessingChain {
return true;
}
private void processResource(Resource currentResource) throws AccessDeniedException, UnexpectedException, InvalidParameterException,
ContentNotAvailableException, InsufficientResourcesException, UnsupportedRequestException, ServiceNotConfiguredException, InterruptedException {
/*
* log source extract from WebLab resource, this value has been
* added by the folderCrawler we just use classic WebLab helpers
* (Annotator) to read source annotation
*/
DublinCoreAnnotator dublinCoreAnnotator = new DublinCoreAnnotator(currentResource);
String source = dublinCoreAnnotator.readSource().firstTypedValue();
this.logger.info("Processing document: " + source);
/*
* call tika to extract text and meta in this resource
*/
ProcessArgs processArgs = new ProcessArgs();
processArgs.setResource(currentResource);
ProcessReturn processReturn = this.tika.process(processArgs);
/*
* Set normalized content as exposed one if none
*/
WProcessingAnnotator wpa = new WProcessingAnnotator(processReturn.getResource());
if(!wpa.readExposedAs().hasValue()){
String content = wpa.readNativeContent().toString();
if (content != null && content.startsWith("http://localhost:8080")){
// set address as a relative one
wpa.writeExposedAs(content.substring(21));
private void initChains() throws JAXBException {
try{
if (chainManager == null){
String url = System.getProperty(CHAIN_REGISTRY);
if (url == null){
throw new JAXBException("Chain registry property is not defined. Please add a system property for "+CHAIN_REGISTRY);
}
this.logger.info("Loading Chains from registry "+url);
UrlResource resource = new UrlResource(url);
XmlBeanFactory factory = new XmlBeanFactory(resource);
chainManager = factory.getBean("chain-manager", HashMap.class);
if (chainManager == null){
throw new IllegalArgumentException("Chain Registry "+url+" do not have a valid chain-manager Initialisation failed..");
}
for(Entry<String, Chain> entry:chainManager.entrySet()){
this.logger.info("Initializing chain with usage context : "+entry.getKey());
entry.getValue().init(this.logger, this.ctx);
}
}
}catch (Exception e) {
this.logger.log(Level.SEVERE,e.getMessage(),e);
throw new JAXBException("Chain initialisation failed : "+e.getMessage(),e);
}
/*
* create the two named entity extractor runnable
*/
final Resource splitResource = processReturn.getResource();
WebLabRunnable gazRunnable = createAnalyserRunnable(splitResource, this.simpleGazetteer);
WebLabRunnable gateRunnable = createAnalyserRunnable(splitResource, this.ngramj, this.gate);
/*
* create the runnable managers and launch both thread
*/
ExecutorService T800 = Executors.newFixedThreadPool(2);
T800.execute(gazRunnable);
T800.execute(gateRunnable);
T800.shutdown();
/*
* wait for the termination
*/
T800.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
/*
* merge resources
*/
ResourcesMerger rm = ResourcesMerger.getInstance();
currentResource = rm.merge(splitResource, gazRunnable.getFinalResource(), gateRunnable.getFinalResource());
/*
* log document format and title extract from WebLab resource, this
* value has been added by the tika. we just use classic WebLab
* helpers (Annotator) to read document annotation
*/
// we have to create another annotator because the variable
// currentResource have changed
dublinCoreAnnotator = new DublinCoreAnnotator(currentResource);
Value<String> readFormat = dublinCoreAnnotator.readFormat();
String format = "No format";
if (readFormat != null) {
format = readFormat.firstTypedValue();
}
Value<String> readTitle = dublinCoreAnnotator.readTitle();
String title = "No title";
if (readTitle != null) {
title = readTitle.firstTypedValue();
}
/*
* just log some information extract from Tika
*/
this.logger.info("Title: " + title + " - Format: " + format);
/*
* index into SOLR
*/
IndexArgs indexArgs = new IndexArgs();
indexArgs.setResource(currentResource);
this.solr.index(indexArgs);
this.logger.info("File: " + source + " indexed.");
/*
* Save into repository
*/
SaveResourceArgs saveResArgs = new SaveResourceArgs();
saveResArgs.setResource(currentResource);
repository.saveResource(saveResArgs);
this.logger.info("File: " + source + " saved in repository.");
}
private void initServices() throws JAXBException {
this.folderQueueManager = JbiServiceFactory.getProxy(QueueManager.class, new QName("http://listener.webservices.weblab.ow2.org/",
"DefaultFolderListenerService"), this.ctx, this.logger);