Unverified Commit 76ed38d9 authored by Alberto Nale's avatar Alberto Nale Committed by GitHub
Browse files

[KNOWAGE-6894][KNOWAGE-6895][KNOWAGE-6896][KNOWAGE-7111][KNOWAGE-7114]

KNOWAGE-6894
KNOWAGE-6895
KNOWAGE-6896
Data Preparation implementation should be fully finished now, it needs to be confirmed working on latest1 because we dont have spark on our local env.

KNOWAGE-7111
Added an additional check inside Metaweb.

KNOWAGE-7114
Implemented custom formulas as requested inside Metaweb and QBE.
Changed KnCalculated field so it reads formulas from a property instead a descriptor, a new required property propCalcFieldFunctions needs to be passed to the component.
parents 38694d0a 29f04523
......@@ -3,3 +3,11 @@ export interface IKnCalculatedField {
type?: string
formula?: string
}
export interface IKnCalculatedFieldFunction {
category: string
formula: string
label: string
name: string
help: string
}
......@@ -85,10 +85,11 @@
</template>
<script lang="ts">
import { PropType } from 'vue'
import { AxiosResponse } from 'axios'
import { createValidations } from '@/helpers/commons/validationHelper'
import { defineComponent } from 'vue'
import { IKnCalculatedField } from '@/components/functionalities/KnCalculatedField/KnCalculatedField'
import { IKnCalculatedField, IKnCalculatedFieldFunction } from '@/components/functionalities/KnCalculatedField/KnCalculatedField'
import { VCodeMirror } from 'vue3-code-mirror'
import Dropdown from 'primevue/dropdown'
......@@ -109,7 +110,8 @@ export default defineComponent({
descriptor: Object,
template: {} as any,
valid: Boolean,
source: String
source: String,
propCalcFieldFunctions: { type: Array as PropType<IKnCalculatedFieldFunction[]>, required: true }
},
data() {
return {
......@@ -132,12 +134,14 @@ export default defineComponent({
},
v$: useValidate() as any,
formulaValidationInterval: {} as any,
isValidFormula: false
isValidFormula: false,
calcFieldFunctions: [] as IKnCalculatedFieldFunction[]
}
},
emits: ['save', 'cancel', 'update:readOnly'],
created() {
this.availableFunctions = [...this.descriptor?.availableFunctions].sort((a, b) => {
this.calcFieldFunctions = [...this.propCalcFieldFunctions]
this.availableFunctions = [...this.calcFieldFunctions].sort((a, b) => {
return a.name.localeCompare(b.name)
})
this.availableFunctions.forEach((x) => {
......@@ -186,7 +190,7 @@ export default defineComponent({
this.selectedCategory = ''
},
filterFunctions() {
let tmp = [...this.descriptor?.availableFunctions].sort((a, b) => {
let tmp = [...this.calcFieldFunctions].sort((a, b) => {
return a.name.localeCompare(b.name)
})
tmp.forEach((x) => {
......@@ -201,7 +205,7 @@ export default defineComponent({
handleOptions() {
let tmp = [] as any
this.descriptor?.availableFunctions
this.calcFieldFunctions
.sort((a, b) => {
return a.name.localeCompare(b.name)
})
......
......@@ -16,7 +16,7 @@
<template #header>
<span>{{ $t('metaweb.businessModel.title') }}</span>
</template>
<BusinessModelTab :propMeta="meta" :observer="observer" :metaUpdated="metaUpdated" @metaUpdated="onMetaUpdated" />
<BusinessModelTab :businessModelId="businessModel.dataSourceId" :propMeta="meta" :observer="observer" :metaUpdated="metaUpdated" @metaUpdated="onMetaUpdated" />
</TabPanel>
<TabPanel>
<template #header>
......
......@@ -91,7 +91,7 @@
<span>{{ $t('metaweb.businessModel.tabView.calcField') }}</span>
</template>
<div :style="mainDescriptor.style.absoluteScroll">
<CalculatedField :selectedBusinessModel="selectedBusinessModel" :propMeta="meta" @metaUpdated="$emit('metaUpdated')" :observer="observer" />
<CalculatedField :selectedBusinessModel="selectedBusinessModel" :propMeta="meta" :propCustomFunctions="customFunctions" @metaUpdated="$emit('metaUpdated')" :observer="observer" />
</div>
</TabPanel>
<TabPanel>
......@@ -191,7 +191,7 @@ export default defineComponent({
MetawebPhysicalTableTab,
MetawebFilterTab
},
props: { propMeta: { type: Object }, observer: { type: Object }, metaUpdated: { type: Boolean } },
props: { propMeta: { type: Object }, observer: { type: Object }, metaUpdated: { type: Boolean }, businessModelId: Number },
emits: ['loading', 'metaUpdated'],
computed: {},
data() {
......@@ -200,6 +200,7 @@ export default defineComponent({
mainDescriptor,
meta: null as any,
menuButtons: [] as any,
customFunctions: [] as any,
showBusinessClassDialog: false,
showBusinessViewDialog: false,
selectedBusinessModel: {} as iBusinessModel,
......@@ -219,6 +220,7 @@ export default defineComponent({
this.loadMeta()
this.createMenuItems()
this.loadRoles()
this.loadCustomFunctions()
},
methods: {
showMenu(event) {
......@@ -264,6 +266,15 @@ export default defineComponent({
await this.$http.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + '2.0/roles').then((response: AxiosResponse<any>) => (this.roles = response.data))
this.loading = false
},
async loadCustomFunctions() {
this.loading = true
await this.$http.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `2.0/configs/KNOWAGE.CUSTOMIZED_DATABASE_FUNCTIONS/${this.businessModelId}`).then((response: AxiosResponse<any>) => {
if (response.data.data && response.data.data.length > 0) {
this.customFunctions = response.data.data.map((funct) => ({ category: 'CUSTOM', formula: funct.value, label: funct.label, name: funct.name, help: 'dataPreparation.custom' }))
} else this.customFunctions = null
})
this.loading = false
},
async onRowReorder(event: any) {
this.loading = true
const postData = { data: { index: event.dragIndex, direction: event.dropIndex - event.dragIndex }, diff: generate(this.observer) }
......
......@@ -15,7 +15,19 @@
</Column>
</DataTable>
<KnCalculatedField v-model:template="selectedCalcField" v-model:visibility="calcFieldDialogVisible" :fields="calcFieldColumns" :descriptor="calcFieldDescriptor" :source="'QBE'" :readOnly="false" :valid="true" @save="onCalcFieldSave" @cancel="calcFieldDialogVisible = false">
<KnCalculatedField
v-if="calcFieldDialogVisible"
v-model:template="selectedCalcField"
v-model:visibility="calcFieldDialogVisible"
:fields="calcFieldColumns"
:descriptor="calcFieldDescriptor"
:propCalcFieldFunctions="calcFieldFunctions"
:source="'QBE'"
:readOnly="false"
:valid="true"
@save="onCalcFieldSave"
@cancel="calcFieldDialogVisible = false"
>
<template #additionalInputs>
<div class="p-field p-col-4">
<span class="p-float-label ">
......@@ -37,6 +49,7 @@
import { AxiosResponse } from 'axios'
import { defineComponent, PropType } from 'vue'
import { iBusinessModel } from '../../../Metaweb'
import { IKnCalculatedFieldFunction } from '@/components/functionalities/KnCalculatedField/KnCalculatedField'
import DataTable from 'primevue/datatable'
import Column from 'primevue/column'
import descriptor from './MetawebCalculatedFieldDescriptor.json'
......@@ -45,11 +58,12 @@ import KnCalculatedField from '@/components/functionalities/KnCalculatedField/Kn
import Dropdown from 'primevue/dropdown'
const { generate, applyPatch } = require('fast-json-patch')
const deepcopy = require('deepcopy')
export default defineComponent({
name: 'metaweb-filter-tab',
components: { DataTable, Column, KnCalculatedField, Dropdown },
props: { selectedBusinessModel: { type: Object as PropType<iBusinessModel | null> }, propMeta: { type: Object }, observer: { type: Object, required: true } },
props: { selectedBusinessModel: { type: Object as PropType<iBusinessModel | null> }, propMeta: { type: Object }, propCustomFunctions: { type: Array }, observer: { type: Object, required: true } },
emits: ['metaUpdated'],
data() {
return {
......@@ -60,18 +74,33 @@ export default defineComponent({
calcFieldDialogVisible: false,
readOnly: false,
selectedCalcField: {} as any,
calcFieldColumns: [] as any
calcFieldColumns: [] as any,
calcFieldFunctions: [] as IKnCalculatedFieldFunction[]
}
},
computed: {
isGeographicBm(): boolean {
let hideFields = false
this.businessModel?.properties?.forEach((el: any) => {
const key = Object.keys(el)[0]
if (key === 'structural.tabletype' && el[key].value === 'geographic dimension') {
hideFields = true
} else hideFields = false
})
return hideFields
}
},
watch: {
selectedBusinessModel() {
this.loadMeta()
this.loadBusinessModel()
this.calcFieldFunctions = this.createCalcFieldFunctions(calcFieldDescriptor.availableFunctions, this.propCustomFunctions)
}
},
created() {
this.loadMeta()
this.loadBusinessModel()
this.calcFieldFunctions = this.createCalcFieldFunctions(calcFieldDescriptor.availableFunctions, this.propCustomFunctions)
},
methods: {
loadMeta() {
......@@ -167,6 +196,23 @@ export default defineComponent({
})
.catch(() => {})
.finally(() => generate(this.observer))
},
createCalcFieldFunctions(providedFunctions, customFunctions?) {
let functions = deepcopy(providedFunctions)
if (customFunctions) {
customFunctions.forEach((funct) => {
functions.push(funct)
})
}
if (!this.isGeographicBm) {
let tempFunctions = deepcopy(functions)
functions = tempFunctions.filter((funct) => {
return funct.category !== 'SPATIAL'
})
}
return functions
}
}
})
......
......@@ -2,121 +2,141 @@
<Card class="p-m-2">
<template #content>
<form class="p-fluid p-formgrid p-grid">
<div class="p-field p-col-6">
<div class="p-field p-col-4">
<Button :label="$t('managers.datasetManagement.monitoring')" class="kn-button kn-button--primary" @click="showMonitoringDialog = true" />
</div>
<div class="p-field p-col-6">
<div class="p-field p-col-4">
<Button :label="$t('managers.datasetManagement.openDP')" class="kn-button kn-button--primary" @click="prepareForDataPreparation" />
</div>
</form>
</template>
</Card>
<MonitoringDialog :visibility="showMonitoringDialog" :dataset="selectedDataset" @close="showMonitoringDialog = false" />
<MonitoringDialog :visibility="showMonitoringDialog" :dataset="selectedDataset" @close="showMonitoringDialog = false" @save="updateDatasetAndSave" />
</template>
<script lang="ts">
import { defineComponent } from 'vue'
import { AxiosResponse } from 'axios'
import descriptor from './DatasetManagementPreparedDataset.json'
import Card from 'primevue/card'
import MonitoringDialog from '@/modules/workspace/dataPreparation/DataPreparationMonitoring/DataPreparationMonitoringDialog.vue'
import { defineComponent } from 'vue'
import { AxiosResponse } from 'axios'
import descriptor from './DatasetManagementPreparedDataset.json'
import Card from 'primevue/card'
import MonitoringDialog from '@/modules/workspace/dataPreparation/DataPreparationMonitoring/DataPreparationMonitoringDialog.vue'
export default defineComponent({
components: { Card, MonitoringDialog },
props: { selectedDataset: { type: Object as any }, dataSources: { type: Array as any } },
emits: ['touched'],
data() {
return {
descriptor,
dataset: {} as any,
availableDatasets: [] as any,
avroDatasets: [] as any,
showMonitoringDialog: false
}
},
created() {
export default defineComponent({
components: { Card, MonitoringDialog },
props: { selectedDataset: { type: Object as any }, dataSources: { type: Array as any } },
emits: ['touched'],
data() {
return {
descriptor,
dataset: {} as any,
availableDatasets: [] as any,
avroDatasets: [] as any,
showMonitoringDialog: false
}
},
created() {
this.dataset = this.selectedDataset
},
watch: {
selectedDataset() {
this.dataset = this.selectedDataset
}
},
methods: {
async loadDataset(datasetId: Number) {
await this.$http
.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `1.0/datasets/dataset/id/${datasetId}`)
.then((response: AxiosResponse<any>) => {
this.dataset = response.data[0]
})
.catch(() => {})
},
watch: {
selectedDataset() {
this.dataset = this.selectedDataset
}
routeToDataPreparation() {
let path = ''
this.$confirm.require({
header: this.$t('managers.datasetManagement.openDP'),
message: this.$t('managers.datasetManagement.confirmMsg'),
icon: 'pi pi-exclamation-triangle',
accept: () => {
this.$router.push(path)
}
})
},
methods: {
routeToDataPreparation() {
let path = ''
this.$confirm.require({
header: this.$t('managers.datasetManagement.openDP'),
message: this.$t('managers.datasetManagement.confirmMsg'),
icon: 'pi pi-exclamation-triangle',
accept: () => {
this.$router.push(path)
}
})
},
async getAllAvroDataSets() {
await this.$http
.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `3.0/datasets/avro`)
.then((response: AxiosResponse<any>) => {
this.avroDatasets = response.data
})
.catch(() => {})
},
async getAllAvroDataSets() {
await this.$http
.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `3.0/datasets/avro`)
.then((response: AxiosResponse<any>) => {
this.avroDatasets = response.data
})
.catch(() => {})
},
async prepareForDataPreparation() {
await this.getAllAvroDataSets()
await this.openDataPreparation(this.selectedDataset)
},
async prepareForDataPreparation() {
await this.getAllAvroDataSets()
await this.openDataPreparation(this.selectedDataset)
},
isAvroReady(dsId: Number) {
if (this.avroDatasets.indexOf(dsId) >= 0 || (dsId && this.avroDatasets.indexOf(dsId.toString())) >= 0) return true
else return false
},
isAvroReady(dsId: Number) {
if (this.avroDatasets.indexOf(dsId) >= 0 || (dsId && this.avroDatasets.indexOf(dsId.toString())) >= 0) return true
else return false
},
openDataPreparation(dataset: any) {
if (dataset.dsTypeCd == 'Prepared') {
//edit existing data prep
this.$http.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `3.0/datasets/advanced/${dataset.id}`).then(
(response: AxiosResponse<any>) => {
let instanceId = response.data.configuration.dataPrepInstanceId
this.$http.get(process.env.VUE_APP_DATA_PREPARATION_PATH + `1.0/process/by-instance-id/${instanceId}`).then(
(response: AxiosResponse<any>) => {
let transformations = response.data.definition
let processId = response.data.id
let datasetId = response.data.instance.dataSetId
if (this.isAvroReady(datasetId))
// check if Avro file has been deleted or not
this.$router.push({ name: 'data-preparation', params: { id: datasetId, transformations: JSON.stringify(transformations), processId: processId, instanceId: instanceId, dataset: JSON.stringify(dataset) } })
else {
this.$store.commit('setInfo', {
title: 'Avro file is missing',
msg: 'Generate it again and then retry'
})
}
},
() => {
this.$store.commit('setError', { title: 'Save error', msg: 'Cannot create process' })
openDataPreparation(dataset: any) {
if (dataset.dsTypeCd == 'Prepared') {
//edit existing data prep
this.$http.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `3.0/datasets/advanced/${dataset.id}`).then(
(response: AxiosResponse<any>) => {
let instanceId = response.data.configuration.dataPrepInstanceId
this.$http.get(process.env.VUE_APP_DATA_PREPARATION_PATH + `1.0/process/by-instance-id/${instanceId}`).then(
(response: AxiosResponse<any>) => {
let transformations = response.data.definition
let processId = response.data.id
let datasetId = response.data.instance.dataSetId
if (this.isAvroReady(datasetId))
// check if Avro file has been deleted or not
this.$router.push({ name: 'data-preparation', params: { id: datasetId, transformations: JSON.stringify(transformations), processId: processId, instanceId: instanceId, dataset: JSON.stringify(dataset) } })
else {
this.$store.commit('setInfo', {
title: 'Avro file is missing',
msg: 'Generate it again and then retry'
})
}
)
},
() => {
this.$store.commit('setError', {
title: 'Cannot open data preparation'
})
}
)
} else if (this.isAvroReady(dataset.id)) {
// original dataset already exported in Avro
this.$router.push({ name: 'data-preparation', params: { id: dataset.id } })
} else {
this.$store.commit('setInfo', {
title: 'Avro file is missing',
msg: 'Generate it again and then retry'
})
}
},
() => {
this.$store.commit('setError', { title: 'Save error', msg: 'Cannot create process' })
}
)
},
() => {
this.$store.commit('setError', {
title: 'Cannot open data preparation'
})
}
)
} else if (this.isAvroReady(dataset.id)) {
// original dataset already exported in Avro
this.$router.push({ name: 'data-preparation', params: { id: dataset.id } })
} else {
this.$store.commit('setInfo', {
title: 'Avro file is missing',
msg: 'Generate it again and then retry'
})
}
},
async updateDatasetAndSave(newConfig) {
this.showMonitoringDialog = false
await this.$http.patch(process.env.VUE_APP_DATA_PREPARATION_PATH + '1.0/instance/' + newConfig.instanceId, { config: newConfig.config }, { headers: { Accept: 'application/json, */*' } }).then(
() => {
this.loadDataset(this.selectedDataset.id)
},
() => {
this.$store.commit('setError', { title: this.$t('common.error.saving'), msg: this.$t('managers.workspaceManagement.dataPreparation.errors.updatingSchedulation') })
}
)
}
})
}
})
</script>
......@@ -118,7 +118,19 @@
<QBESavingDialog :visible="savingDialogVisible" :propDataset="qbe" @close="savingDialogVisible = false" @datasetSaved="$emit('datasetSaved')" />
<QBEJoinDefinitionDialog v-if="joinDefinitionDialogVisible" :visible="joinDefinitionDialogVisible" :qbe="qbe" :propEntities="entities?.entities" :id="uniqueID" :selectedQuery="selectedQuery" @close="onJoinDefinitionDialogClose"></QBEJoinDefinitionDialog>
<KnCalculatedField v-model:template="selectedCalcField" v-model:visibility="calcFieldDialogVisible" :fields="calcFieldColumns" :descriptor="calcFieldDescriptor" :readOnly="false" :valid="true" source="QBE" @save="onCalcFieldSave" @cancel="calcFieldDialogVisible = false">
<KnCalculatedField
v-if="calcFieldDialogVisible"
v-model:template="selectedCalcField"
v-model:visibility="calcFieldDialogVisible"
:fields="calcFieldColumns"
:descriptor="calcFieldDescriptor"
:propCalcFieldFunctions="calcFieldFunctions"
:readOnly="false"
:valid="true"
source="QBE"
@save="onCalcFieldSave"
@cancel="calcFieldDialogVisible = false"
>
<template #additionalInputs>
<div class="p-field" :class="[selectedCalcField.type === 'DATE' ? 'p-col-3' : 'p-col-4']">
<span class="p-float-label ">
......@@ -259,6 +271,7 @@ export default defineComponent({
calcFieldDialogVisible: false,
calcFieldColumns: [] as any,
selectedCalcField: null as any,
calcFieldFunctions: [] as any,
colors: ['#D7263D', '#F46036', '#2E294E', '#1B998B', '#C5D86D', '#3F51B5', '#8BC34A', '#009688', '#F44336']
}
},
......@@ -309,6 +322,7 @@ export default defineComponent({
this.loading = false
},
async loadQBE() {
this.loadCalcFieldFunctions()
await this.initializeQBE()
await this.loadCustomizedDatasetFunctions()
await this.loadEntities()
......@@ -331,6 +345,9 @@ export default defineComponent({
this.mainQuery = this.qbe?.qbeJSONQuery?.catalogue?.queries[0]
this.selectedQuery = this.qbe?.qbeJSONQuery?.catalogue?.queries[0]
},
loadCalcFieldFunctions() {
this.calcFieldFunctions = calcFieldDescriptor.availableFunctions
},
getQBEFromModel() {
if (!this.dataset) return {}
......@@ -396,7 +413,15 @@ export default defineComponent({
},
async loadCustomizedDatasetFunctions() {
const id = this.dataset?.dataSourceId ? this.dataset.dataSourceId : this.qbe?.qbeDataSourceId
await this.$http.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `2.0/configs/KNOWAGE.CUSTOMIZED_DATABASE_FUNCTIONS/${id}`).then((response: AxiosResponse<any>) => (this.customizedDatasetFunctions = response.data))
await this.$http.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `2.0/configs/KNOWAGE.CUSTOMIZED_DATABASE_FUNCTIONS/${id}`).then((response: AxiosResponse<any>) => {
this.customizedDatasetFunctions = response.data
if (response.data.data && response.data.data.length > 0) {
let customFunctions = response.data.data.map((funct) => ({ category: 'CUSTOM', formula: funct.value, label: funct.label, name: funct.name, help: 'dataPreparation.custom' }))
customFunctions.forEach((funct) => {
this.calcFieldFunctions.push(funct)
})
}
})
},
async loadEntities() {
const datamartName = this.dataset?.dataSourceId ? this.dataset.name : this.qbe?.qbeDatamarts
......
<template>
<div class="kn-page kn-data-preparation">
<KnCalculatedField v-model:visibility="showCFDialog" @save="saveCFDialog" @cancel="cancelCFDialog" :fields="columns" :descriptor="cfDescriptor" :readOnly="readOnly" @update:readOnly="updateReadOnly" v-model:template="selectedTransformation" :valid="cfType !== ''">
<KnCalculatedField
v-model:visibility="showCFDialog"
@save="saveCFDialog"
@cancel="cancelCFDialog"
:fields="columns"
:descriptor="cfDescriptor"
:propCalcFieldFunctions="cfDescriptor.availableFunctions"
:readOnly="readOnly"
@update:readOnly="updateReadOnly"
v-model:template="selectedTransformation"
:valid="cfType !== ''"
>