Commit 6ce52344 authored by Matteo Massarotto's avatar Matteo Massarotto
Browse files

Merge branch 'data-preparation-2'

parents ebcbbd02 237b20fe
......@@ -506,6 +506,16 @@ public class DataSetResource extends AbstractDataSetResource {
return super.deleteDataset(label);
}
@DELETE
@Path("/id/{id}")
@UserConstraint(functionalities = { SpagoBIConstants.SELF_SERVICE_DATASET_MANAGEMENT })
public Response deleteDatasetById(@PathParam("id") int id) {
IDataSetDAO datasetDao = DAOFactory.getDataSetDAO();
IDataSet dataset = datasetDao.loadDataSetById(id);
String label = dataset.getLabel();
return super.deleteDataset(label);
}
/**
* Delete a version for the selected dataset.
*
......
......@@ -1072,8 +1072,8 @@ public abstract class AbstractDataSetResource extends AbstractSpagoBIResource {
Response response = restClient.target(serviceUrlAsURL + "/knowage-data-preparation/api/1.0/instance/" + instanceId).request()
.header("X-Kn-Authorization", token).get();
JSONObject instance = new JSONObject(response.readEntity(String.class));
String sourceDsLabel = instance.getString("dataSetLabel");
deleteAvroFolder(sourceDsLabel);
int sourceDsId = instance.getInt("dataSetId");
deleteAvroFolder(sourceDsId);
// delete data preparation process instance
restClient.target(serviceUrlAsURL + "/knowage-data-preparation/api/1.0/instance/" + instanceId).request().header("X-Kn-Authorization", token)
.delete();
......@@ -1085,9 +1085,10 @@ public abstract class AbstractDataSetResource extends AbstractSpagoBIResource {
return Response.ok().build();
}
private void deleteAvroFolder(String label) {
private void deleteAvroFolder(int dsId) {
try {
Path avroExportFolder = Paths.get(SpagoBIUtilities.getResourcePath(), "dataPreparation", (String) getUserProfile().getUserId(), label);
Path avroExportFolder = Paths.get(SpagoBIUtilities.getResourcePath(), "dataPreparation", (String) getUserProfile().getUserId(),
Integer.toString(dsId));
Files.walkFileTree(avroExportFolder, new SimpleFileVisitor<Path>() {
// delete directories or folders
......
......@@ -247,7 +247,8 @@ public class AvroExportJob extends AbstractExportJob {
@Override
protected OutputStream getDataOutputStream() {
try {
avroExportFolder = Paths.get(resourcePathAsStr, "dataPreparation", (String) userProfile.getUserId(), dataSet.getLabel());
String dsIdasString = Integer.toString(dataSet.getId());
avroExportFolder = Paths.get(resourcePathAsStr, "dataPreparation", (String) userProfile.getUserId(), dsIdasString);
Files.createDirectories(avroExportFolder);
return Files.newOutputStream(avroExportFolder.resolve(data));
} catch (Exception e) {
......
......@@ -401,13 +401,14 @@ public class DataSetResource {
*
*/
@GET
@Path("/advanced/{label}")
@Path("/advanced/{dsId}")
@Produces(MediaType.APPLICATION_JSON)
@UserConstraint(functionalities = { SpagoBIConstants.SELF_SERVICE_DATASET_MANAGEMENT })
public SbiDataSet getAdvancedDataSet(@PathParam("label") String label) {
public SbiDataSet getAdvancedDataSet(@PathParam("dsId") int dsId) {
try {
SbiDataSet dataSet = DAOFactory.getSbiDataSetDAO().loadSbiDataSetByLabel(label);
final UserProfile userProfile = getUserProfile();
SbiDataSet dataSet = DAOFactory.getSbiDataSetDAO().loadSbiDataSetByIdAndOrganiz(dsId, userProfile.getOrganization());
return dataSet;
} catch (Exception t) {
......@@ -424,8 +425,8 @@ public class DataSetResource {
@Path("/avro")
@Produces(MediaType.APPLICATION_JSON)
@UserConstraint(functionalities = { SpagoBIConstants.SELF_SERVICE_DATASET_MANAGEMENT })
public List<String> getPreparedDataSets() {
List<String> preparedDataSets = new ArrayList<String>();
public List<String> getAvroDataSets() {
List<String> avroDataSets = new ArrayList<String>();
try {
final UserProfile userProfile = getUserProfile();
java.nio.file.Path avroExportFolder = Paths.get(SpagoBIUtilities.getRootResourcePath(), userProfile.getOrganization(), "dataPreparation",
......@@ -434,14 +435,14 @@ public class DataSetResource {
for (int i = 0; i < datasets.length; i++) {
boolean avroReady = new File(datasets[i], "ready").exists();
if (avroReady) {
preparedDataSets.add(datasets[i].getName());
avroDataSets.add(datasets[i].getName());
}
}
} catch (Exception e) {
logger.error("Cannot get list of prepared datasets", e);
logger.error("Cannot get list of Avro datasets", e);
return new ArrayList<String>();
}
return preparedDataSets;
return avroDataSets;
}
/**
......
......@@ -69,8 +69,8 @@
methods: {
async loadLogs() {
if (this.dataset && this.dataset.label) {
await this.$http.get(process.env.VUE_APP_DATA_PREPARATION_PATH + '1.0/process/by-destination-data-set/' + this.dataset.label).then((response: AxiosResponse<any>) => {
if (this.dataset && this.dataset.id) {
await this.$http.get(process.env.VUE_APP_DATA_PREPARATION_PATH + '1.0/process/by-destination-data-set/' + this.dataset.id).then((response: AxiosResponse<any>) => {
let instance = response.data.instance
if (instance) {
this.instanceId = instance.id
......
......@@ -69,180 +69,180 @@
</template>
<script lang="ts">
import { defineComponent, PropType } from 'vue'
import { defineComponent, PropType } from 'vue'
import { createValidations } from '@/helpers/commons/validationHelper'
import { AxiosResponse } from 'axios'
import Dialog from 'primevue/dialog'
import Textarea from 'primevue/textarea'
import DataPreparationDescriptor from './DataPreparationDescriptor.json'
import useValidate from '@vuelidate/core'
import DataPreparationValidationDescriptor from './DataPreparationValidationDescriptor.json'
import KnValidationMessages from '@/components/UI/KnValidatonMessages.vue'
import { IDataPreparationDataset, IDataPreparationColumn } from '@/modules/workspace/dataPreparation/DataPreparation'
import { createValidations } from '@/helpers/commons/validationHelper'
import { AxiosResponse } from 'axios'
import Dialog from 'primevue/dialog'
import Textarea from 'primevue/textarea'
import DataPreparationDescriptor from './DataPreparationDescriptor.json'
import useValidate from '@vuelidate/core'
import DataPreparationValidationDescriptor from './DataPreparationValidationDescriptor.json'
import KnValidationMessages from '@/components/UI/KnValidatonMessages.vue'
import { IDataPreparationDataset, IDataPreparationColumn } from '@/modules/workspace/dataPreparation/DataPreparation'
import dataPreparationMonitoringDescriptor from '@/modules/workspace/dataPreparation/DataPreparationMonitoring/DataPreparationMonitoringDescriptor.json'
import KnScheduler from '@/components/UI/KnScheduler/KnScheduler.vue'
import dataPreparationMonitoringDescriptor from '@/modules/workspace/dataPreparation/DataPreparationMonitoring/DataPreparationMonitoringDescriptor.json'
import KnScheduler from '@/components/UI/KnScheduler/KnScheduler.vue'
export default defineComponent({
name: 'data-preparation-detail-save-dialog',
props: {
originalDataset: {} as any,
config: {} as any,
columns: [] as PropType<IDataPreparationColumn[]>,
instanceId: {} as any,
processId: {} as any,
preparedDsMeta: {} as any,
visibility: Boolean
export default defineComponent({
name: 'data-preparation-detail-save-dialog',
props: {
originalDataset: {} as any,
config: {} as any,
columns: [] as PropType<IDataPreparationColumn[]>,
instanceId: {} as any,
processId: {} as any,
preparedDsMeta: {} as any,
visibility: Boolean
},
components: { Dialog, KnScheduler, KnValidationMessages, Textarea },
data() {
return {
descriptor: DataPreparationDescriptor,
preparedDataset: {} as IDataPreparationDataset,
v$: useValidate() as any,
validationDescriptor: DataPreparationValidationDescriptor,
schedulerDescriptor: dataPreparationMonitoringDescriptor,
currentCronExpression: '',
isFirstSave: true,
touched: false,
schedulationPaused: false,
schedulationEnabled: false
}
},
updated() {
if (this.processId && this.processId != '') this.isFirstSave = false
},
emits: ['update:visibility', 'update:instanceId', 'update:processId'],
validations() {
return {
preparedDataset: createValidations('preparedDataset', this.validationDescriptor.validations.configuration)
}
},
computed: {
saveButtonDisabled(): any {
return this.v$.$invalid || !this.preparedDataset.name
}
},
methods: {
savePreparedDataset(): void {
let processDefinition = this.createProcessDefinition()
this.saveOrUpdateProcess(processDefinition).then(
(response: AxiosResponse<any>) => {
let processId = response.data.id
this.$emit('update:processId', processId)
let datasetDefinition = this.createDatasetDefinition()
this.saveOrUpdateInstance(processId, datasetDefinition).then(
(response: AxiosResponse<any>) => {
this.$emit('update:instanceId', response.data.id)
this.$store.commit('setInfo', { title: 'Saved successfully' })
},
() => {
this.$store.commit('setError', { title: 'Save error', msg: 'Cannot add process instance' })
}
)
},
() => {
this.$store.commit('setError', { title: 'Save error', msg: 'Cannot create process' })
}
)
this.resetAndClose()
},
components: { Dialog, KnScheduler, KnValidationMessages, Textarea },
data() {
return {
descriptor: DataPreparationDescriptor,
preparedDataset: {} as IDataPreparationDataset,
v$: useValidate() as any,
validationDescriptor: DataPreparationValidationDescriptor,
schedulerDescriptor: dataPreparationMonitoringDescriptor,
currentCronExpression: '',
isFirstSave: true,
touched: false,
schedulationPaused: false,
schedulationEnabled: false
}
saveOrUpdateProcess(processDefinition) {
if (this.processId && this.processId != '') return this.$http.put(process.env.VUE_APP_DATA_PREPARATION_PATH + `1.0/process/${this.processId}`, processDefinition)
else return this.$http.post(process.env.VUE_APP_DATA_PREPARATION_PATH + '1.0/process', processDefinition)
},
updated() {
if (this.processId && this.processId != '') this.isFirstSave = false
saveOrUpdateInstance(processId, datasetDefinition) {
if (this.instanceId && this.instanceId != '') return this.$http.post(process.env.VUE_APP_DATA_PREPARATION_PATH + `1.0/instance/${this.instanceId}`, datasetDefinition)
else return this.$http.post(process.env.VUE_APP_DATA_PREPARATION_PATH + '1.0/process/' + processId + '/instance', datasetDefinition)
},
emits: ['update:visibility', 'update:instanceId', 'update:processId'],
createDatasetDefinition() {
let toReturn = {}
toReturn['config'] = {}
toReturn['config']['paused'] = this.schedulationPaused
if (this.schedulationEnabled) toReturn['config']['cron'] = this.currentCronExpression
validations() {
return {
preparedDataset: createValidations('preparedDataset', this.validationDescriptor.validations.configuration)
toReturn['dataSetLabel'] = this.originalDataset.label
toReturn['dataSetId'] = this.originalDataset.id
var d = new Date()
if (this.preparedDataset.label) {
toReturn['destinationDataSetLabel'] = this.preparedDataset.label
} else {
toReturn['destinationDataSetLabel'] = 'ds__' + (d.getTime() % 10000000)
}
toReturn['destinationDataSetName'] = this.preparedDataset.name
toReturn['destinationDataSetDescription'] = this.preparedDataset.description
toReturn['meta'] = this.createMetaDefinition()
toReturn['dsId'] = this.preparedDataset.id
return toReturn
},
computed: {
saveButtonDisabled(): any {
return this.v$.$invalid || !this.preparedDataset.name
}
createMetaDefinition() {
let meta = [] as Array<any>
this.columns?.forEach((col) => {
let item = {}
item['displayedName'] = col.fieldAlias
item['name'] = col.header
item['fieldType'] = col.fieldType
item['type'] = col.Type
meta.push(item)
})
return meta
},
methods: {
savePreparedDataset(): void {
let processDefinition = this.createProcessDefinition()
this.saveOrUpdateProcess(processDefinition).then(
(response: AxiosResponse<any>) => {
let processId = response.data.id
this.$emit('update:processId', processId)
let datasetDefinition = this.createDatasetDefinition()
this.saveOrUpdateInstance(processId, datasetDefinition).then(
(response: AxiosResponse<any>) => {
this.$emit('update:instanceId', response.data.id)
this.$store.commit('setInfo', { title: 'Saved successfully' })
},
() => {
this.$store.commit('setError', { title: 'Save error', msg: 'Cannot add process instance' })
}
)
},
() => {
this.$store.commit('setError', { title: 'Save error', msg: 'Cannot create process' })
}
)
this.resetAndClose()
},
saveOrUpdateProcess(processDefinition) {
if (this.processId && this.processId != '') return this.$http.put(process.env.VUE_APP_DATA_PREPARATION_PATH + `1.0/process/${this.processId}`, processDefinition)
else return this.$http.post(process.env.VUE_APP_DATA_PREPARATION_PATH + '1.0/process', processDefinition)
},
saveOrUpdateInstance(processId, datasetDefinition) {
if (this.instanceId && this.instanceId != '') return this.$http.patch(process.env.VUE_APP_DATA_PREPARATION_PATH + `1.0/instance/${this.instanceId}`, datasetDefinition)
else return this.$http.patch(process.env.VUE_APP_DATA_PREPARATION_PATH + '1.0/process/' + processId + '/instance', datasetDefinition)
},
createDatasetDefinition() {
let toReturn = {}
toReturn['config'] = {}
toReturn['config']['paused'] = this.schedulationPaused
if (this.schedulationEnabled) toReturn['config']['cron'] = this.currentCronExpression
toReturn['dataSetLabel'] = this.originalDataset.label
var d = new Date()
if (this.preparedDataset.label) {
toReturn['destinationDataSetLabel'] = this.preparedDataset.label
}
else {
toReturn['destinationDataSetLabel'] = 'ds__' + (d.getTime() % 10000000)
}
toReturn['destinationDataSetName'] = this.preparedDataset.name
toReturn['destinationDataSetDescription'] = this.preparedDataset.description
toReturn['meta'] = this.createMetaDefinition()
toReturn['dsId'] = this.preparedDataset.id
return toReturn
},
createMetaDefinition() {
let meta = [] as Array<any>
this.columns?.forEach((col) => {
let item = {}
item['displayedName'] = col.fieldAlias
item['name'] = col.header
item['fieldType'] = col.fieldType
item['type'] = col.Type
meta.push(item)
})
return meta
},
createProcessDefinition() {
let toReturn = {}
if (this.config && this.config.transformations) toReturn['definition'] = this.config.transformations
return toReturn
},
resetAndClose(): void {
this.closeDialog()
},
closeDialog(): void {
this.$emit('update:visibility', false)
},
loadTranslations(): void {
this.descriptor.dataPreparation.refreshRate.options.forEach((element) => {
element.name = this.$t(element.name)
})
},
updateSchedulationPaused(newSchedulationPaused) {
this.schedulationPaused = newSchedulationPaused
},
updateSchedulationEnabled(newSchedulationEnabled) {
this.schedulationEnabled = newSchedulationEnabled
},
updateCurrentCronExpression(newCronExpression) {
this.currentCronExpression = newCronExpression
}
createProcessDefinition() {
let toReturn = {}
if (this.config && this.config.transformations) toReturn['definition'] = this.config.transformations
return toReturn
},
resetAndClose(): void {
this.closeDialog()
},
closeDialog(): void {
this.$emit('update:visibility', false)
},
loadTranslations(): void {
this.descriptor.dataPreparation.refreshRate.options.forEach((element) => {
element.name = this.$t(element.name)
})
},
updateSchedulationPaused(newSchedulationPaused) {
this.schedulationPaused = newSchedulationPaused
},
updateSchedulationEnabled(newSchedulationEnabled) {
this.schedulationEnabled = newSchedulationEnabled
},
updateCurrentCronExpression(newCronExpression) {
this.currentCronExpression = newCronExpression
}
},
watch: {
preparedDsMeta: {
handler() {
if (Object.keys(this.preparedDsMeta).length > 0) {
this.preparedDataset = this.preparedDsMeta
this.currentCronExpression = this.preparedDsMeta.config?.cron ? this.preparedDsMeta.config.cron : ''
this.schedulationPaused = this.preparedDsMeta.config?.schedulationPaused || false
watch: {
preparedDsMeta: {
handler() {
if (Object.keys(this.preparedDsMeta).length > 0) {
this.preparedDataset = this.preparedDsMeta
this.currentCronExpression = this.preparedDsMeta.config?.cron ? this.preparedDsMeta.config.cron : ''
this.schedulationEnabled = this.preparedDsMeta.config?.cron ? true : false
}
},
deep: true
}
},
this.schedulationPaused = this.preparedDsMeta.config?.schedulationPaused || false
created() {
this.loadTranslations()
this.schedulationEnabled = this.preparedDsMeta.config?.cron ? true : false
}
},
deep: true
}
})
},
created() {
this.loadTranslations()
}
})
</script>
<style lang="scss">
.dataPreparationSaveDialog {
min-width: 600px !important;
width: 600px !important;
max-width: 600px !important;
}
.dataPreparationSaveDialog {
min-width: 600px !important;
width: 600px !important;
max-width: 600px !important;
}
</style>
......@@ -193,7 +193,7 @@ export default defineComponent({
async loadDataset(datasetLabel: string) {
this.loading = true
await this.$http
.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `1.0/datasets/${datasetLabel}`)
.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `1.0/datasets/id/${datasetId}`)
.then((response: AxiosResponse<any>) => {
this.selectedDataset = response.data[0]
})
......@@ -259,7 +259,7 @@ export default defineComponent({
this.creationMenuButtons.push({ key: '0', label: this.$t('workspace.myData.prepareData'), visible: true })
},
async previewDataset(dataset: any) {
await this.loadDataset(dataset.label)
await this.loadDataset(dataset.id)
this.previewDialogVisible = true
},
editDataset() {
......@@ -272,17 +272,17 @@ export default defineComponent({
openDataPreparation(dataset: any) {
if (dataset.dsTypeCd == 'Prepared') {
//edit existing data prep
this.$http.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `3.0/datasets/advanced/${dataset.label}`).then(
this.$http.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `3.0/datasets/advanced/${dataset.id}`).then(
(response: AxiosResponse<any>) => {
let instanceId = response.data.configuration.dataPrepInstanceId
this.$http.get(process.env.VUE_APP_DATA_PREPARATION_PATH + `1.0/process/by-instance-id/${instanceId}`).then(
(response: AxiosResponse<any>) => {
let transformations = response.data.definition
let processId = response.data.id
let datasetLabel = response.data.instance.dataSetLabel
if (this.isAvroReady(datasetLabel))
let datasetId = response.data.instance.dataSetId
if (this.isAvroReady(datasetId))
// check if Avro file has been deleted or not
this.$router.push({ name: 'data-preparation', params: { id: datasetLabel, transformations: JSON.stringify(transformations), processId: processId, instanceId: instanceId, dataset: JSON.stringify(dataset) } })
this.$router.push({ name: 'data-preparation', params: { id: datasetId, transformations: JSON.stringify(transformations), processId: processId, instanceId: instanceId, dataset: JSON.stringify(dataset) } })
else {
this.$store.commit('setInfo', {
title: 'Avro file is missing',
......@@ -301,9 +301,9 @@ export default defineComponent({
})
}
)
} else if (this.isAvroReady(dataset.label)) {
} else if (this.isAvroReady(dataset.id)) {
// original dataset already exported in Avro
this.$router.push({ name: 'data-preparation', params: { id: dataset.label } })
this.$router.push({ name: 'data-preparation', params: { id: dataset.id } })
} else {
this.$store.commit('setInfo', {
title: 'Avro file is missing',
......@@ -311,8 +311,8 @@ export default defineComponent({
})
}
},
isAvroReady(dsLabel: String) {
if (this.avroDatasets.indexOf(dsLabel) >= 0) return true
isAvroReady(dsId: Number) {
if (this.avroDatasets.indexOf(dsId) >= 0 || (dsId && this.avroDatasets.indexOf(dsId.toString())) >= 0) return true
else return false
},
async getAllAvroDataSets() {
......@@ -379,7 +379,7 @@ export default defineComponent({
this.loading = false
},
async cloneDataset(dataset: any) {
await this.loadDataset(dataset.label)
await this.loadDataset(dataset.id)
this.cloneDialogVisible = true
},
async handleDatasetClone(dataset: any) {
......@@ -400,7 +400,7 @@ export default defineComponent({
})
},
datasetPreparation(dataset: any) {
this.$router.push({ name: 'data-preparation', params: { id: dataset.label } })
this.$router.push({ name: 'data-preparation', params: { id: dataset.id } })
},
deleteDatasetConfirm(dataset: any) {
......@@ -455,9 +455,9 @@ export default defineComponent({
async updateDatasetAndSave(newConfig) {
this.showMonitoring = false
await this.$http.patch(process.env.VUE_APP_DATA_PREPARATION_PATH + '1.0/instance/' + newConfig.instanceId, { config: newConfig.config }).then(
await this.$http.patch(process.env.VUE_APP_DATA_PREPARATION_PATH + '1.0/instance/' + newConfig.instanceId, { config: newConfig.config },{ headers: { Accept: 'application/json, */*'} }).then(
() => {
this.loadDataset(this.selectedDataset.label)
this.loadDataset(this.selectedDataset.id)
},
() => {
this.$store.commit('setError', { title: this.$t('common.error.saving'), msg: this.$t('managers.workspaceManagement.dataPreparation.errors.updatingSchedulation') })
......
......@@ -62,7 +62,7 @@
:key="index"
:viewType="'dataset'"
:document="dataset"
:isAvroReady="isAvroReady(dataset.label)"
:isAvroReady="isAvroReady(dataset.id)"