Commit 0e8590ca authored by Arnaud Saval's avatar Arnaud Saval
Browse files

WEBLAB-1000 use stable version of services and update processing chain logging...

WEBLAB-1000 use stable version of services and update processing chain logging with more information

git-svn-id: svn://svn.forge.objectweb.org/svnroot/weblab/trunk/WebLabApplications/bundle@3481 a803c184-d8de-4a0e-a52b-d3eef88212b6
parent ff09c0c8
......@@ -2,7 +2,7 @@
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>1.2.5.1-SNAPSHOT</id>
<id>1.2.6-SNAPSHOT</id>
<formats>
<format>zip</format>
</formats>
......
......@@ -5,7 +5,7 @@
<parent>
<groupId>org.ow2.weblab.bundle</groupId>
<artifactId>versions</artifactId>
<version>1.2.5.1-SNAPSHOT</version>
<version>1.2.6-SNAPSHOT</version>
<relativePath>../versions/pom.xml</relativePath>
</parent>
......
......@@ -7,14 +7,14 @@
<parent>
<groupId>org.ow2.weblab.bundle</groupId>
<artifactId>versions</artifactId>
<version>1.2.5.1-SNAPSHOT</version>
<version>1.2.6-SNAPSHOT</version>
<relativePath>../../versions/pom.xml</relativePath>
</parent>
<name>su-CHAIN-provide</name>
<artifactId>su-CHAIN-provide</artifactId>
<groupId>org.ow2.weblab.bundle.petals</groupId>
<version>1.0</version>
<version>1.1-SNAPSHOT</version>
<packaging>jbi-service-unit</packaging>
<description></description>
......
......@@ -7,14 +7,14 @@
<parent>
<groupId>org.ow2.weblab.bundle</groupId>
<artifactId>versions</artifactId>
<version>1.2.5.1-SNAPSHOT</version>
<version>1.2.6-SNAPSHOT</version>
<relativePath>../../versions/pom.xml</relativePath>
</parent>
<name>su-POJO-ChainService-provide</name>
<artifactId>su-POJO-ChainService-provide</artifactId>
<groupId>org.ow2.weblab.bundle.petals</groupId>
<version>1.0</version>
<version>1.1-SNAPSHOT</version>
<packaging>jbi-service-unit</packaging>
<description></description>
......
......@@ -19,9 +19,12 @@ package org.ow2.weblab;
import java.lang.management.ManagementFactory;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
......@@ -37,6 +40,10 @@ import javax.xml.transform.dom.DOMResult;
import org.ow2.petals.component.framework.api.exception.PEtALSCDKException;
import org.ow2.petals.component.framework.api.message.Exchange;
import org.ow2.weblab.bundle.server.Chain;
import org.ow2.weblab.bundle.server.Chainable;
import org.ow2.weblab.bundle.server.Sequence;
import org.ow2.weblab.bundle.server.Service;
import org.ow2.weblab.bundle.server.Split;
import org.ow2.weblab.chain.ChainRequestType;
import org.ow2.weblab.chain.ChainResponse;
import org.ow2.weblab.chain.ObjectFactory;
......@@ -62,7 +69,9 @@ public class ProcessingChain implements ProcessingChainMBean {
private ComponentContext ctx;
/**
* Map of current loaded chains
*/
private HashMap<String, Chain> chainManager;
......@@ -198,4 +207,74 @@ public class ProcessingChain implements ProcessingChainMBean {
}
@Override
public void setEnable(String serviceToChange, boolean enabled) {
if (serviceToChange == null){
return;
}
// retrieve service
for(Chain chain:chainManager.values()){
for(Service service: retrieveServicesFrom(chain.getAnalysers())){
if (serviceToChange.equals(service.getWebLabService().getServiceName())){
service.getWebLabService().setEnabled(enabled);
return;
}
}
}
}
@Override
public Map<String, Boolean> status() {
// retrieve services from chains
Map<String, Boolean> status = new HashMap<String, Boolean>();
for(Chain chain:chainManager.values()){
for(Service service: retrieveServicesFrom(chain.getAnalysers())){
status.put(service.getWebLabService().getServiceName(), service.getWebLabService().isEnabled());
}
}
return status;
}
/**
* Retrieve list of service and their status from a chainable
* @param chainable a chainable being o r containing a service
* @return a set of services
*/
private Set<Service> retrieveServicesFrom(Chainable chainable) {
Set<Service> services = new HashSet<Service>();
if (chainable instanceof Service){
services.add((Service)chainable);
}else if (chainable instanceof Split){
Split split = (Split)chainable;
for(Chainable subchainable:split.getServices()){
services.addAll(retrieveServicesFrom(subchainable));
}
}else if (chainable instanceof Sequence){
Sequence sequence = (Sequence )chainable;
for(Chainable subchainable:sequence.getServices()){
services.addAll(retrieveServicesFrom(subchainable));
}
}else{
logger.warning("Unsupported chainable type : "+chainable.getClass());
}
return services;
}
@Override
public Map<String, Integer> servicesFailures() {
Map<String, Integer> status = new HashMap<String, Integer>();
for(Chain chain:chainManager.values()){
for(Service service: retrieveServicesFrom(chain.getAnalysers())){
status.put(service.getWebLabService().getServiceName(), service.getWebLabService().getFailed());
}
}
return status;
}
}
\ No newline at end of file
......@@ -18,10 +18,42 @@
package org.ow2.weblab;
import java.util.List;
import java.util.Map;
/**
* MBean to manage processing chains
*/
public interface ProcessingChainMBean {
/**
* Reload chains (need to delete and restart a chain)
*/
public void reload();
/**
* return list of chains id
* @return chains ids
*/
public List<String> listChainsId();
/**
* Enables or disables a service.
* A disabled service will act as an identity service.
* @param service a string describing the service: serviceNS+ServiceName
* @param enabled if true, it will enable the service, else it will disable it.
*/
public void setEnable(String service, boolean enabled);
/**
* Return if services are enabled or disabled.
* @return services status.
*/
public Map<String, Boolean> status();
/**
* Return how many times services are failing to execute consecutively.
* @return services failing.
*/
public Map<String, Integer> servicesFailures();
}
......@@ -115,14 +115,14 @@ public class Chain {
logger.info("Context is " + usageContext);
// crawler
Resource resource = queuemanager.execute(null, usageContext);
Resource resource = queuemanager.execute(null, usageContext, logger);
if (resource == null){
logger.warning("Skipping resource from quemanager since it is null for " + usageContext);
return;
}
// normalizer
resource = normalizers.execute(resource, usageContext);
resource = normalizers.execute(resource, usageContext, logger);
DublinCoreAnnotator dublinCoreAnnotator = new DublinCoreAnnotator(resource);
String source = dublinCoreAnnotator.readSource().firstTypedValue();
......@@ -141,7 +141,7 @@ public class Chain {
}
// analyser
resource = analysers.execute(resource, usageContext);
resource = analysers.execute(resource, usageContext, logger);
// we have to create another annotator because the variable
// currentResource have changed
......@@ -164,11 +164,11 @@ public class Chain {
logger.info("Title: " + title + " - Format: " + format);
// indexes
resource = indexers.execute(resource, usageContext);
resource = indexers.execute(resource, usageContext, logger);
logger.info("File: " + source + " indexed.");
// repositories
resource = repositories.execute(resource, usageContext);
resource = repositories.execute(resource, usageContext, logger);
logger.info("File: " + source + " saved in repository.");
}
......@@ -184,7 +184,7 @@ public class Chain {
}
@Override
public Resource execute(Resource resource, String usageContext)
public Resource execute(Resource resource, String usageContext, Logger logger)
throws ContentNotAvailableException, UnexpectedException,
EmptyQueueException, InsufficientResourcesException,
AccessDeniedException, ServiceNotConfiguredException,
......
......@@ -38,9 +38,8 @@ import org.ow2.weblab.core.services.UnsupportedRequestException;
*/
public interface Chainable {
public void init(final Logger logger, final ComponentContext ctx) throws Exception;
public Resource execute(final Resource resource, final String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException;
public Resource execute(final Resource resource, final String usageContext, final Logger logger) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException;
}
......@@ -59,14 +59,14 @@ public class Sequence implements Chainable{
}
@Override
public Resource execute(final Resource resource, final String usageContext)
public Resource execute(final Resource resource, final String usageContext, final Logger logger)
throws ContentNotAvailableException, UnexpectedException,
EmptyQueueException, InsufficientResourcesException,
AccessDeniedException, ServiceNotConfiguredException,
UnsupportedRequestException, InvalidParameterException {
Resource currentResource = resource;
for(Chainable service:services){
currentResource = service.execute(currentResource, usageContext);
currentResource = service.execute(currentResource, usageContext, logger);
}
return currentResource;
}
......
......@@ -78,6 +78,9 @@ public class Service implements Chainable{
return this.handler;
}
public WebLabService getWebLabService(){
return this.service;
}
/**
* init Services
* @param logger a logger
......@@ -89,20 +92,20 @@ public class Service implements Chainable{
}
if("QueueManager".equals(handler.getInterfaceName())){
service = new WebLabService() {
service = new WebLabService(handler.getServiceNs(), handler.getServiceName()) {
QueueManager queuemanager = JbiServiceFactory.getProxy(QueueManager.class, new QName(handler.getServiceNs(), handler.getServiceName()), ctx, logger);
@Override
public Resource execute(Resource resource, String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
public Resource toExecute(Resource resource, String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
NextResourceArgs nra = new NextResourceArgs();
nra.setUsageContext(usageContext);
return queuemanager.nextResource(nra).getResource();
}
};
}else if("Analyser".equals(handler.getInterfaceName())){
service = new WebLabService() {
service = new WebLabService(handler.getServiceNs(), handler.getServiceName()) {
Analyser analyser = JbiServiceFactory.getProxy(Analyser.class, new QName(handler.getServiceNs(), handler.getServiceName()), ctx, logger);
@Override
public Resource execute(Resource resource, String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
public Resource toExecute(Resource resource, String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
ProcessArgs processArgs =new ProcessArgs();
processArgs.setResource(resource);
processArgs.setUsageContext(usageContext);
......@@ -110,11 +113,11 @@ public class Service implements Chainable{
}
};
}else if("Indexer".equals(handler.getInterfaceName())){
service = new WebLabService() {
service = new WebLabService(handler.getServiceNs(), handler.getServiceName()) {
Indexer indexer = JbiServiceFactory.getProxy(Indexer.class, new QName(handler.getServiceNs(), handler.getServiceName()), ctx, logger);
@Override
public Resource execute(Resource resource, String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
public Resource toExecute(Resource resource, String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
IndexArgs indexArgs =new IndexArgs();
indexArgs.setResource(resource);
//indexArgs.setUsageContext(usageContext); // BUG
......@@ -123,11 +126,11 @@ public class Service implements Chainable{
}
};
}else if("ResourceContainer".equals(handler.getInterfaceName())){
service = new WebLabService() {
service = new WebLabService(handler.getServiceNs(), handler.getServiceName()) {
ResourceContainer container = JbiServiceFactory.getProxy(ResourceContainer.class, new QName(handler.getServiceNs(), handler.getServiceName()), ctx, logger);
@Override
public Resource execute(Resource resource, String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
public Resource toExecute(Resource resource, String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
SaveResourceArgs saveArgs =new SaveResourceArgs();
saveArgs.setResource(resource);
saveArgs.setUsageContext(usageContext);
......@@ -168,8 +171,8 @@ public class Service implements Chainable{
}
@Override
public Resource execute(Resource resource, String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
return service.execute(resource, usageContext);
public Resource execute(Resource resource, String usageContext, Logger logger) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException {
return service.execute(resource, usageContext, logger);
}
}
......@@ -24,7 +24,6 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.jbi.component.ComponentContext;
......@@ -66,7 +65,7 @@ public class Split implements Chainable{
}
@Override
public Resource execute(final Resource resource, final String usageContext)
public Resource execute(final Resource resource, final String usageContext, final Logger logger)
throws ContentNotAvailableException, UnexpectedException,
EmptyQueueException, InsufficientResourcesException,
AccessDeniedException, ServiceNotConfiguredException,
......@@ -81,7 +80,7 @@ public class Split implements Chainable{
ExecutorService T800 = Executors.newFixedThreadPool(services.size());
Set<WebLabRunnable> runnables = new HashSet<WebLabRunnable>();
for(Chainable service:services){
runnables.add(createAnalyserRunnable(splitResource, usageContext, null, service));
runnables.add(createAnalyserRunnable(splitResource, usageContext, logger, service));
}
for(WebLabRunnable runnable:runnables){
T800.execute(runnable);
......@@ -128,9 +127,9 @@ public class Split implements Chainable{
Resource resource = input;
for (Chainable service : services) {
try {
resource = service.execute(resource, usageContext);
resource = service.execute(resource, usageContext, logger);
} catch (Exception e) {
logger.log(Level.SEVERE, e.getLocalizedMessage(), e);
logger.warning(e.getLocalizedMessage());
}
}
finalResource = resource;
......
......@@ -17,6 +17,8 @@
*/
package org.ow2.weblab.bundle.server;
import java.util.logging.Logger;
import org.ow2.weblab.core.model.Resource;
import org.ow2.weblab.core.services.AccessDeniedException;
import org.ow2.weblab.core.services.ContentNotAvailableException;
......@@ -27,8 +29,105 @@ import org.ow2.weblab.core.services.ServiceNotConfiguredException;
import org.ow2.weblab.core.services.UnexpectedException;
import org.ow2.weblab.core.services.UnsupportedRequestException;
public interface WebLabService {
/**
* Represent a WebLab web service
* @author asaval
*
*/
public abstract class WebLabService{
/**
* Maximum failure threshold
*/
private static final int THRESHOLD = 3;
/**
* Number of consecutive service failure
*/
private int failed = 0;
private String serviceName;
// allows to enable or disable a service
protected boolean enabled = true;
public WebLabService(String serviceNs, String serviceName) {
this.serviceName = "{"+serviceNs+"}"+serviceName;
}
public boolean isEnabled() {
return enabled;
}
// enable or disable a service and reset the failed counter
public void setEnabled(boolean enabled) {
this.enabled = enabled;
this.failed = 0;
}
public Resource execute(final Resource resource, final String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException;
public int getFailed(){
return this.failed;
}
public String getServiceName(){
return this.serviceName;
}
/**
* Ask a service to process/return a resource
* @param resource
* @param usageContext
* @param logger
* @return a resource
* @throws ContentNotAvailableException
* @throws UnexpectedException
* @throws EmptyQueueException
* @throws InsufficientResourcesException
* @throws AccessDeniedException
* @throws ServiceNotConfiguredException
* @throws UnsupportedRequestException
* @throws InvalidParameterException
*/
public Resource execute(final Resource resource, final String usageContext, final Logger logger) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException{
Resource result = resource;
if (this.enabled && failed < THRESHOLD){
try {
result = toExecute(resource, usageContext);
// reset failed counter.
this.failed = 0;
}finally{
if (result == resource && result != null){
// increment the "service does nothing"
this.failed++;
if (failed > 1){
// the service is doing nothing ...
logger.warning(serviceName+" failed "+this.failed+" time(s) to process the resource "+result.getUri());
}
if (this.failed == THRESHOLD){
logger.warning("Failing service "+serviceName+" is now disabled.");
}
}
}
}
return result;
}
/**
* Abstract method to implement for each service
* @param resource
* @param usageContext
* @return
* @throws ContentNotAvailableException
* @throws UnexpectedException
* @throws EmptyQueueException
* @throws InsufficientResourcesException
* @throws AccessDeniedException
* @throws ServiceNotConfiguredException
* @throws UnsupportedRequestException
* @throws InvalidParameterException
*/
protected abstract Resource toExecute(final Resource resource, final String usageContext) throws ContentNotAvailableException, UnexpectedException, EmptyQueueException, InsufficientResourcesException, AccessDeniedException, ServiceNotConfiguredException, UnsupportedRequestException, InvalidParameterException;
}
......@@ -9,14 +9,14 @@
<parent>
<groupId>org.ow2.weblab.bundle</groupId>
<artifactId>versions</artifactId>
<version>1.2.5.1-SNAPSHOT</version>
<version>1.2.6-SNAPSHOT</version>
<relativePath>../../versions/pom.xml</relativePath>
</parent>
<name>su-SOAP-ChainService-consume</name>
<artifactId>su-SOAP-ChainService-consume</artifactId>
<groupId>org.ow2.weblab.bundle.petals</groupId>
<version>1.0</version>
<version>1.1-SNAPSHOT</version>
<packaging>jbi-service-unit</packaging>
<description></description>
......
......@@ -59,7 +59,7 @@
<!-- reading route -->
<route>
<from uri="direct:weblab-default-out"/>
<setHeader headerName="uriOptions"><constant>&amp;initialDelay=100&amp;delay=100</constant></setHeader>
<setHeader headerName="uriOptions"><constant>&amp;initialDelay=50&amp;delay=50&amp;noop=true&amp;readLock=none</constant></setHeader>
<process ref="CallEndpointFromURIProcessor"></process>
<to uri="ref:file-endpoint" />
<process ref="ResourceToURIProcessor"></process>
......
......@@ -44,4 +44,5 @@ file.path=data/content
# CamelContentManager properties #
################################
# Camel configuration file
#camel.config.file=conf/camelBeans.xml
\ No newline at end of file
......@@ -4,7 +4,7 @@
<parent>
<groupId>org.ow2.weblab.bundle</groupId>
<artifactId>versions</artifactId>
<version>1.2.5.1-SNAPSHOT</version>
<version>1.2.6-SNAPSHOT</version>
<relativePath>../../versions/pom.xml</relativePath>
</parent>
......@@ -19,9 +19,9 @@
<dependencies>
<dependency>
<groupId>org.ow2.weblab.incubation</groupId>
<groupId>org.ow2.weblab.components</groupId>
<artifactId>camel-content-manager</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.0.0</version>
<type>jar</type>
</dependency>
......