Commit 6e7719d2 authored by Yann Mombrun's avatar Yann Mombrun
Browse files

OPENAXES-189 - Release new version 1.1.0 of OpenAXES

OPENAXES-188 - Enable the camel component to annotate the created document with its language if a header weblab:dc:language is present.
OPENAXES-184 - Enable French support to OpenAXES
parent 694db39e
......@@ -68,9 +68,12 @@ You can stop OpenAXES using the command:
If you want to process new data, you have to move video files in the data/toIndex folder. The metadata could be provided aside.
Automatic Speech Recognition is done in English by default.
If your data is in French you can either provide a metadata file that states 'fr' in language metadata, or create a folder named fr inside to index folder and put your data in.
The folder data/toIndex-sample contains some video samples with associated metadata that can be used as bootstrap samples.
The best is to do that prior to start the openaxes system to prevent from polling an unfinished file.
The process might last around 5 times the duration of the video.
The Hawtio Console available at http://localhost:8282/hawtio (login/pass: weblab/weblab) can be used to monitor the process.
......@@ -86,7 +89,7 @@ Know issues and evolution requests at https://jira.ow2.org/issues/?jql=project%2
The version 1.0.1 of the Open AXES is available for download at http://forge.ow2.org/project/showfiles.php?group_id=436.
The version 1.1.0 of the Open AXES is available for download at http://forge.ow2.org/project/showfiles.php?group_id=436.
Open AXES is an output of the AXES European project.
The goal AXES (http://axes-project.eu) was to develop tools that provide various types of users with new engaging ways to interact with audiovisual libraries,
......@@ -95,14 +98,14 @@ the "Open AXES" solution gathers innovative audiovisual content analyses technol
as well as an ergonomic interface to navigate the video archive.
This version 1.0.1 contains:
This version 1.1.0 contains:
- a folder gathering service, provided by Airbus Defence and Space, enabling to collect video files and metadata (in various format: json, properties, rdf) in a set of relevant folders;
- a shot-detector and keyframe-extraction service provided by Technicolor and enabling to select some keyframes representative of each shot of the video
(and prevent image based components to analyse every single frame of a video or picking random ones) which is available from binaries and only allowed for non commercial use;
- a video-normaliser service, provided by Airbus Defence and Space and based on FFmpeg, enabling to convert the original video in appropriate formats for other components and UI;
- an image-classifier service, provided by the Katholieke Universiteit Leuven in charge of classifying selected images over a set of more than 1000 classes
which is available from binaries and allowed for non commercial use only;
- an automated speech recognition service, provided by Airbus Defence and Space and based on CMU Sphinx, extracting the text from English audio speech;
- an automated speech recognition service, provided by Airbus Defence and Space and based on CMU Sphinx, extracting the text from English or French (using LIUM models) audio speech;
- a named entity detection service, provided by Airbus Defence and Space and based on GATE platform, extracting organisation, persons and locations from the text of the speeches;
- a text and metadata indexing service as well as the search and fuse search interfaces, named LIMAS and provided by the University of Twente,
saving the results of the processes and being the backend server for the UI, also in charge of calling the other search engines and fusing their result.
......
......@@ -31,8 +31,7 @@
<prop key="mongodb.db">${weblab.data}mongodb</prop>
<prop key="weblab.repository">${weblab.data}repository</prop>
<prop key="weblab.files">${weblab.data}toIndex?initialDelay=10000&amp;delay=60000&amp;maxMessagesPerPoll=50&amp;recursive=false&amp;antInclude=*.mp4,*.mpg,*.mpeg,*.webm,*.avi,*.flv,*.ts,*.wmv,*.mov,*.ogv,*.m4v&amp;noop=false
</prop>
<prop key="weblab.files">${weblab.data}toIndex?initialDelay=10000&amp;delay=60000&amp;maxMessagesPerPoll=50&amp;recursive=true&amp;antInclude=**/*.mp4,**/*.mpg,**/*.mpeg,**/*.webm,**/*.avi,**/*.flv,**/*.ts,**/*.wmv,**/*.mov,**/*.ogv,**/*.m4v&amp;noop=false</prop>
<prop key="collection.name">cOpenAXES</prop>
<prop key="weblab.components">${weblab.home}components</prop>
......
......@@ -6,7 +6,7 @@
<import resource="classpath:META-INF/cxf/cxf.xml" />
<import resource="classpath:META-INF/cxf/cxf-servlet.xml" />
<jaxws:endpoint id="sphinxEndpoint" implementor="#sphinxAnalyser" address="/analyser">
<jaxws:endpoint id="sphinxEndpointEn" implementor="#sphinxAnalyserEn" address="/analyserEn">
<jaxws:properties>
<entry key="exceptionMessageCauseEnabled" value="true" />
<entry key="faultStackTraceEnabled" value="true" />
......@@ -25,7 +25,7 @@
</jaxws:dataBinding>
</jaxws:endpoint>
<bean id="sphinxAnalyser" class="org.ow2.weblab.service.transcript.sphinx.SphinxTranscriptorService">
<bean id="sphinxAnalyserEn" class="org.ow2.weblab.service.transcript.sphinx.SphinxTranscriptorService">
<constructor-arg type="org.ow2.weblab.service.transcript.sphinx.ATranscriptCreator">
<bean class="eu.axes.services.sphinx.AXESTranscriptCreator">
<constructor-arg type="org.ow2.weblab.service.transcript.sphinx.SphinxTranscriber">
......@@ -45,4 +45,43 @@
</constructor-arg>
</bean>
<jaxws:endpoint id="sphinxEndpointFr" implementor="#sphinxAnalyserFr" address="/analyserFr">
<jaxws:properties>
<entry key="exceptionMessageCauseEnabled" value="true" />
<entry key="faultStackTraceEnabled" value="true" />
<entry key="schema-validation-enabled" value="true" />
</jaxws:properties>
<jaxws:dataBinding>
<bean class="org.apache.cxf.jaxb.JAXBDataBinding">
<property name="namespaceMap">
<map>
<entry key="http://weblab.ow2.org/core/1.2/model#" value="wl" />
<entry key="http://weblab.ow2.org/core/1.2/services/analyser" value="an" />
<entry key="http://weblab.ow2.org/core/1.2/services/exception" value="ex" />
</map>
</property>
</bean>
</jaxws:dataBinding>
</jaxws:endpoint>
<bean id="sphinxAnalyserFr" class="org.ow2.weblab.service.transcript.sphinx.SphinxTranscriptorService">
<constructor-arg type="org.ow2.weblab.service.transcript.sphinx.ATranscriptCreator">
<bean class="eu.axes.services.sphinx.AXESTranscriptCreator">
<constructor-arg type="org.ow2.weblab.service.transcript.sphinx.SphinxTranscriber">
<bean class="org.ow2.weblab.service.transcript.sphinx.SphinxTranscriber">
<constructor-arg type="edu.cmu.sphinx.api.Configuration">
<bean class="edu.cmu.sphinx.api.Configuration">
<property name="acousticModelPath" value="resource:/fr/lium/models/acoustic" />
<property name="dictionaryPath" value="resource:/fr/lium/models/dict/frenchWords62K.dic" />
<property name="languageModelPath" value="resource:/fr/lium/models/language/french3g62K.lm.dmp" />
</bean>
</constructor-arg>
<constructor-arg type="java.lang.String" value="file:#{systemProperties['weblab.home']}/conf/SphinxConfiguration.xml" />
</bean>
</constructor-arg>
<constructor-arg type="java.lang.String" value="#{systemProperties['asr.cache']}" />
</bean>
</constructor-arg>
</bean>
</beans>
......@@ -138,7 +138,7 @@ sed 's?BUNDLELOCATION?'`pwd`'?' conf/gui/axes-research.conf.tmpl > conf/gui/axes
sudo ln -sf "$(pwd)/conf/gui/axes-research.conf" /etc/apache2/sites-enabled/axes-research.conf
sed 's?FFMPEG_PATH?'$(pwd)/components/ffmpeg-1.1.16/ffmpeg.sh'?' conf/gui/defaults.py.tmpl > conf/gui/defaults.py.tmpl1
sed 's?BUNDLEVERSION?'1.0.1'?' conf/gui/defaults.py.tmpl1 > conf/gui/defaults.py.tmpl2
sed 's?BUNDLEVERSION?'1.1.0'?' conf/gui/defaults.py.tmpl1 > conf/gui/defaults.py.tmpl2
if [[ -n "$includeOnTheFlyClassifier" ]] ; then
sed 's?VISUAL_SEARCH_ACTIVATED?'\'category-t\''?' conf/gui/defaults.py.tmpl2 > conf/gui/defaults.py
elif [[ -n "$includeClassifier" ]] ; then
......
......@@ -3,7 +3,7 @@
<groupId>eu.axes.bundle.servers</groupId>
<artifactId>celery-manager</artifactId>
<version>1.0.1</version>
<version>1.1.0</version>
<packaging>jar</packaging>
......
......@@ -3,7 +3,7 @@
<groupId>eu.axes.bundle.servers</groupId>
<artifactId>imsearch-manager</artifactId>
<version>1.0.1</version>
<version>1.1.0</version>
<packaging>jar</packaging>
......
......@@ -3,7 +3,7 @@
<groupId>eu.axes.bundle.servers</groupId>
<artifactId>jetty-manager</artifactId>
<version>1.0.1</version>
<version>1.1.0</version>
<packaging>jar</packaging>
......
......@@ -3,7 +3,7 @@
<groupId>eu.axes.bundle.servers</groupId>
<artifactId>mongodb-manager</artifactId>
<version>1.0.1</version>
<version>1.1.0</version>
<packaging>jar</packaging>
......
......@@ -4,7 +4,7 @@
<groupId>eu.axes.bundle.osgi</groupId>
<artifactId>openaxes-camel</artifactId>
<version>1.0.1</version>
<version>1.1.0</version>
<packaging>bundle</packaging>
<name>OpenAXES Camel Chains</name>
......
......@@ -32,11 +32,10 @@ import eu.axes.utils.AxesAnnotator;
/**
* The WebLab producer.
*
* To perform actions on WebLab resources :
* To perform actions on WebLab resources:
* - create a new resource
*
* @author emilienbondu
*
* @author ymombrun, based on emilienbondu's WebLabProducer
*/
public class AXESProducer extends DefaultProducer {
......@@ -57,10 +56,19 @@ public class AXESProducer extends DefaultProducer {
@Override
public void process(final Exchange exchange) {
MessageHelper.copyHeaders(exchange.getIn(), exchange.getOut(), true);
final Object languageOptionalHeader = exchange.getIn().getHeader("weblab:dc:language");
final String language;
if (languageOptionalHeader == null) {
language = null;
} else {
language = languageOptionalHeader.toString();
}
final URI endpointURI = URI.create(this.endpoint.getEndpointUri());
if ((endpointURI.getAuthority() != null) && endpointURI.getAuthority().equalsIgnoreCase("create")) {
try {
this.create(exchange);
this.create(exchange, language);
} catch (final WebLabCheckedException wlce) {
throw new WebLabUncheckedException(wlce);
}
......@@ -72,7 +80,7 @@ public class AXESProducer extends DefaultProducer {
}
private void create(final Exchange exchange) throws WebLabCheckedException {
private void create(final Exchange exchange, final String language) throws WebLabCheckedException {
final File file = this.tryConvertTo(File.class, exchange);
final String path = FilenameUtils.normalize(file.getAbsolutePath());
......@@ -99,6 +107,10 @@ public class AXESProducer extends DefaultProducer {
final Annotation documentAnnot = WebLabResourceFactory.createAndLinkAnnotation(document);
final DublinCoreAnnotator docDCA = this.annotateWithCommonProperties(document, documentAnnot, file, videoId);
docDCA.writeSource(path);
if (language != null) {
docDCA.writeLanguage(language);
}
final WProcessingAnnotator docWPA = new WProcessingAnnotator(URI.create(document.getUri()), documentAnnot);
docWPA.writeOriginalFileName(originalFilename);
docWPA.writeOriginalFileSize(Long.valueOf(size));
......
......@@ -41,7 +41,7 @@
<http-conf:conduit name="*.http-conduit">
<http-conf:client ReceiveTimeout="7200000" />
<http-conf:client ReceiveTimeout="18000000" />
</http-conf:conduit>
......@@ -62,7 +62,14 @@
</cxf:properties>
</cxf:cxfEndpoint>
<cxf:cxfEndpoint id="SphinxRecognitionService" address="http://localhost:${tomcat.http.port}/axes-sphinx-recognition/analyser" serviceClass="org.ow2.weblab.core.services.Analyser">
<cxf:cxfEndpoint id="EnSphinxRecognitionService" address="http://localhost:${tomcat.http.port}/axes-sphinx-recognition/analyserEn" serviceClass="org.ow2.weblab.core.services.Analyser">
<cxf:properties>
<entry key="dataFormat" value="PAYLOAD" />
<entry key="allowStreaming" value="true" />
</cxf:properties>
</cxf:cxfEndpoint>
<cxf:cxfEndpoint id="FrSphinxRecognitionService" address="http://localhost:${tomcat.http.port}/axes-sphinx-recognition/analyserFr" serviceClass="org.ow2.weblab.core.services.Analyser">
<cxf:properties>
<entry key="dataFormat" value="PAYLOAD" />
<entry key="allowStreaming" value="true" />
......@@ -155,9 +162,15 @@
</osgi:service-properties>
</osgi:service>
<osgi:service id="osgi-service-asr" ref="SphinxRecognitionService" interface="org.apache.camel.Endpoint">
<osgi:service id="osgi-service-asr-en" ref="EnSphinxRecognitionService" interface="org.apache.camel.Endpoint">
<osgi:service-properties>
<entry key="name" value="service-asr" />
<entry key="name" value="service-asr-en" />
</osgi:service-properties>
</osgi:service>
<osgi:service id="osgi-service-asr-fr" ref="FrSphinxRecognitionService" interface="org.apache.camel.Endpoint">
<osgi:service-properties>
<entry key="name" value="service-asr-fr" />
</osgi:service-properties>
</osgi:service>
......@@ -208,7 +221,7 @@
<entry key="name" value="service-visor" />
</osgi:service-properties>
</osgi:service>
<osgi:service id="osgi-service-image-indexer" ref="ImageIndexerService" interface="org.apache.camel.Endpoint">
<osgi:service-properties>
<entry key="name" value="service-image-indexer" />
......@@ -230,7 +243,7 @@
<!--======================================================= -->
<!--======================== CAMEL CONTEXT ================ -->
<!--======================================================= -->
<camelContext id="weblabCamelContext" xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="myDeadLetterErrorHandler">
<camelContext id="weblabCamelContext" xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="myDeadLetterErrorHandler" xmlns:dc="http://purl.org/dc/elements/1.1/">
<routeContextRef ref="processingRoutes" />
<routeContextRef ref="fileConsumingRoutes" />
......
......@@ -27,6 +27,12 @@
<setHeader headerName="weblab:wlp:hasOriginalFileSize">
<simple>${header.CamelFileLength}</simple>
</setHeader>
<filter>
<simple>${file:parent} regex '.*/fr$'</simple>
<setHeader headerName="weblab:dc:language">
<constant>fr</constant>
</setHeader>
</filter>
<to uri="axes://create" />
<convertBodyTo type="java.lang.String" />
<inOnly uri="jms:queue:weblabEmptyDocsIn" />
......
......@@ -9,7 +9,7 @@
<!-- ======================================================= -->
<!-- =================== ROUTE CONTEXT ===================== -->
<!-- ======================================================= -->
<routeContext id="processingRoutes">
<routeContext id="processingRoutes" xmlns:dc="http://purl.org/dc/elements/1.1/">
<route id="metadataRoute" streamCache="true" autoStartup="true">
......@@ -54,7 +54,7 @@
<continued>
<constant>true</constant>
</continued>
<log loggingLevel="WARN" message="Error calling metadata service for '${headers.weblab:wlp:hasOriginalFileName}'. Ignoring it and continue." />
<log loggingLevel="WARN" message="Error calling classifier service for '${headers.weblab:wlp:hasOriginalFileName}'. Ignoring it and continue." />
</onException>
<choice>
......@@ -72,7 +72,7 @@
</route>
<route id="asrRoute" streamCache="true" autoStartup="true">
<route id="asrRoute" streamCache="true" autoStartup="true" xmlns:dc="http://purl.org/dc/elements/1.1/">
<from uri="jms:queue:weblabDocsClassified" />
<onException>
......@@ -80,16 +80,24 @@
<continued>
<constant>true</constant>
</continued>
<log loggingLevel="WARN" message="Error calling metadata service for '${headers.weblab:wlp:hasOriginalFileName}'. Ignoring it and continue." />
<log loggingLevel="WARN" message="Error calling asr service for '${headers.weblab:wlp:hasOriginalFileName}'. Ignoring it and continue." />
</onException>
<log message="Calling ASR service for '${headers.weblab:wlp:hasOriginalFileName}'." />
<to uri="weblab:analyser:service-asr" />
<choice>
<when>
<xpath>boolean(//dc:language[text() = 'fr'])</xpath>
<log message="Calling French ASR service for '${headers.weblab:wlp:hasOriginalFileName}'." />
<to uri="weblab:analyser:service-asr-fr" />
</when>
<otherwise>
<log message="Calling English ASR service for '${headers.weblab:wlp:hasOriginalFileName}'." />
<to uri="weblab:analyser:service-asr-en" />
</otherwise>
</choice>
<inOnly uri="jms:queue:weblabDocsWithASR" />
</route>
<route id="nerRoute" streamCache="true" autoStartup="true">
<from uri="jms:queue:weblabDocsWithASR" />
......@@ -98,7 +106,7 @@
<continued>
<constant>true</constant>
</continued>
<log loggingLevel="WARN" message="Error calling metadata service for '${headers.weblab:wlp:hasOriginalFileName}'. Ignoring it and continue." />
<log loggingLevel="WARN" message="Error calling ner service for '${headers.weblab:wlp:hasOriginalFileName}'. Ignoring it and continue." />
</onException>
<log message="Calling NER service for '${headers.weblab:wlp:hasOriginalFileName}'." />
......
......@@ -3,7 +3,7 @@
<groupId>eu.axes.bundle.servers</groupId>
<artifactId>pastec-manager</artifactId>
<version>1.0.1</version>
<version>1.1.0</version>
<packaging>jar</packaging>
......
......@@ -3,7 +3,7 @@
<groupId>eu.axes.bundle.servers</groupId>
<artifactId>visor-manager</artifactId>
<version>1.0.1</version>
<version>1.1.0</version>
<packaging>jar</packaging>
......
......@@ -3,7 +3,7 @@
<groupId>eu.axes.bundle</groupId>
<artifactId>open-axes</artifactId>
<version>1.0.1</version>
<version>1.1.0</version>
<packaging>pom</packaging>
......@@ -45,7 +45,7 @@
<metadata.enricher.version>2.3.3</metadata.enricher.version>
<file.exposer.version>1.2.0</file.exposer.version>
<image.classifier.version>1.0.2</image.classifier.version>
<sphinx.recognition.version>1.1.1</sphinx.recognition.version>
<sphinx.recognition.version>1.2.0</sphinx.recognition.version>
<pastec.indexer.version>0.2.0</pastec.indexer.version>
</properties>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment