Unverified Commit badbf3ce authored by Alberto Nale's avatar Alberto Nale Committed by GitHub
Browse files

GDPR - Dataset Management & QBE

GDPR Has been implemented in the 2 components and seems to be working as expected.
parents 23069d60 a164b837
......@@ -1197,6 +1197,7 @@
"ckanXslSheetNumber": "Sheet Number",
"creationUser": "Creation User",
"dataframeName": "Dataframe variable name",
"decript": "Decript value",
"deleteAllParams": "Clear all dataset parameters",
"deleteAllParamsMsg": "Are you sure you want to delete all dataset parameters?",
"deleteAllRequestHeaderMsg": "Are you sure you want to delete all REST request headers",
......@@ -1238,6 +1239,7 @@
"params": "Parameters",
"peristenceWarning": "Dataset with parameters cannot be persisted!",
"persistTableName": "Persist Table Name",
"personal": "Sensitive data",
"pivotColName": "Category Column to be Pivoted",
"pivotColValue": "Value Column to be Pivoted",
"pivotIsNumRows": "Automatic Columns numeration?",
......@@ -1277,13 +1279,17 @@
"sparqlDefaultGraphIri": "Default Graph IRI",
"sparqlEndpoint": "Sparql Endpoint",
"sparqlExecutionTimeout": "Execution timeout(s)",
"subjectId": "Subject key",
"tableEmpty": "Click on the + icon to insert data.",
"timestampFormat": "Timestamp Format",
"title": "Dataset Management",
"typeOrJsonPathValue": "Type or JSON Path Type",
"viewQbeButton": "View Qbe Query",
"viewQbeMsg": "The dataset should be saved before viewing the QBE query",
"viewQbeWarning": "Cannot open QBE query"
"viewQbeWarning": "Cannot open QBE query",
"openDP": "Open data preparation",
"confirmMsg": "You will lose all unsaved changes, are you sure you want to open Data Preparation?",
"monitoring": "Monitoring"
},
"domainsManagement": {
"domainCode": "Domain code",
......
......@@ -33,7 +33,7 @@
</template>
<TypeCard
:selectedDataset="selectedDataset"
:datasetTypes="datasetTypes"
:datasetTypes="filteredDatasetTypes"
:dataSources="dataSources"
:businessModels="businessModels"
:scriptTypes="scriptTypes"
......@@ -60,7 +60,7 @@
<LinkCard :selectedDataset="selectedDataset" :metaSourceResource="metaSourceResource" :activeTab="activeTab" @addTables="onAddLinkedTables" @removeTables="onRemoveLinkedTables" />
</TabPanel>
<TabPanel>
<TabPanel v-if="selectedDataset.dsTypeCd != 'Prepared'">
<template #header>
<span>{{ $t('cron.advanced') }}</span>
</template>
......@@ -118,6 +118,7 @@ export default defineComponent({
selectedDataset: {} as any,
previewDataset: {} as any,
selectedDatasetVersions: [] as any,
filteredDatasetTypes: [] as any,
scheduling: {
repeatInterval: null as String | null
} as any,
......@@ -167,9 +168,13 @@ export default defineComponent({
await this.getSelectedDataset()
await this.getSelectedDatasetVersions()
this.insertCurrentVersion()
this.filteredDatasetTypes = this.datasetTypes
} else {
this.selectedDataset = { ...detailViewDescriptor.newDataset }
this.selectedDatasetVersions = []
this.filteredDatasetTypes = this.datasetTypes.filter((cd) => {
return cd.VALUE_CD != 'Prepared'
})
}
},
insertCurrentVersion() {
......@@ -299,12 +304,10 @@ export default defineComponent({
}
},
async manageDatasetFieldMetadata(fieldsColumns) {
//Temporary workaround because fieldsColumns is now an object with a new structure after changing DataSetJSONSerializer
if (fieldsColumns.columns != undefined && fieldsColumns.columns != null) {
var columnsArray = new Array()
var columnsNames = new Array()
//create columns list
for (var i = 0; i < fieldsColumns.columns.length; i++) {
var element = fieldsColumns.columns[i]
columnsNames.push(element.column)
......@@ -313,9 +316,10 @@ export default defineComponent({
columnsNames = this.removeDuplicates(columnsNames)
for (i = 0; i < columnsNames.length; i++) {
var columnObject = { displayedName: '', name: '', fieldType: '', type: '' }
var columnObject = { displayedName: '', name: '', fieldType: '', type: '', personal: false, decript: false, subjectId: false }
var currentColumnName = columnsNames[i]
//this will remove the part before the double dot if the column is in the format ex: it.eng.spagobi.Customer:customerId
//remove the part before the double dot if the column is in the format ex: it.eng.spagobi.Customer:customerId
if (currentColumnName.indexOf(':') != -1) {
var arr = currentColumnName.split(':')
columnObject.displayedName = arr[1]
......@@ -331,6 +335,12 @@ export default defineComponent({
columnObject.type = element.pvalue
} else if (element.pname.toUpperCase() == 'fieldType'.toUpperCase()) {
columnObject.fieldType = element.pvalue
} else if (element.pname.toUpperCase() == 'personal'.toUpperCase()) {
columnObject.personal = element.pvalue
} else if (element.pname.toUpperCase() == 'decript'.toUpperCase()) {
columnObject.decript = element.pvalue
} else if (element.pname.toUpperCase() == 'subjectId'.toUpperCase()) {
columnObject.subjectId = element.pvalue
}
}
}
......@@ -338,7 +348,6 @@ export default defineComponent({
}
return columnsArray
// end workaround ---------------------------------------------------
}
},
checkFormulaForParams() {
......
......@@ -19,7 +19,22 @@
</Column>
<Column field="fieldType" :header="$t('managers.datasetManagement.fieldType')" :sortable="true">
<template #body="{data}">
<Dropdown class="kn-material-input" :style="linkTabDescriptor.style.maxwidth" v-model="data.fieldType" :options="fieldMetadataTypes" optionLabel="value" optionValue="value" @change="saveFieldsMetadata" />
<Dropdown class="kn-material-input" :style="linkTabDescriptor.style.maxwidth" v-model="data.fieldType" :options="fieldMetadataTypes" optionLabel="value" optionValue="value" @change="saveFieldsMetadata('fieldType')" />
</template>
</Column>
<Column field="personal" :header="$t('managers.datasetManagement.personal')" :sortable="true">
<template #body="{data}">
<Checkbox id="personal" v-model="data.personal" :binary="true" @change="saveFieldsMetadata('personal')" />
</template>
</Column>
<Column field="decript" :header="$t('managers.datasetManagement.decript')" :sortable="true">
<template #body="{data}">
<Checkbox id="decript" v-model="data.decript" :binary="true" @change="saveFieldsMetadata('decript')" />
</template>
</Column>
<Column field="subjectId" :header="$t('managers.datasetManagement.subjectId')" :sortable="true">
<template #body="{data}">
<Checkbox id="subjectId" v-model="data.subjectId" :binary="true" @change="saveFieldsMetadata('subjectId')" />
</template>
</Column>
</DataTable>
......@@ -38,9 +53,10 @@
import Column from 'primevue/column'
import Message from 'primevue/message'
import Dropdown from 'primevue/dropdown'
import Checkbox from 'primevue/checkbox'
export default defineComponent({
components: { Card, Column, DataTable, Message, Dropdown },
components: { Card, Column, DataTable, Message, Dropdown, Checkbox },
props: {
selectedDataset: { type: Object as any }
},
......@@ -88,7 +104,20 @@
this.fieldsMetadata = fieldsMetadata
},
saveFieldsMetadata() {
saveFieldsMetadata(fieldName) {
this.warnForDuplicateSpatialFields()
this.applyMetadataChangesToFields(fieldName)
},
applyMetadataChangesToFields(fieldName) {
for (let i = 0; i < this.fieldsMetadata.length; i++) {
for (let j = 0; j < this.dataset.meta.columns.length; j++) {
if (this.fieldsMetadata[i].column == this.dataset.meta.columns[j].column && this.dataset.meta.columns[j].pname == fieldName) {
this.dataset.meta.columns[j].pvalue = this.fieldsMetadata[i][fieldName]
}
}
}
},
warnForDuplicateSpatialFields() {
var numberOfSpatialAttribute = 0
for (let i = 0; i < this.fieldsMetadata.length; i++) {
if (this.fieldsMetadata[i].fieldType == 'SPATIAL_ATTRIBUTE') {
......@@ -99,20 +128,6 @@
}
}
}
for (let i = 0; i < this.fieldsMetadata.length; i++) {
for (let j = 0; j < this.dataset.meta.columns.length; j++) {
if (this.fieldsMetadata[i].column == this.dataset.meta.columns[j].column && this.dataset.meta.columns[j].pname == 'fieldType') {
this.dataset.meta.columns[j].pvalue = this.fieldsMetadata[i].fieldType
}
}
}
for (let i = 0; i < this.fieldsMetadata.length; i++) {
for (let j = 0; j < this.dataset.meta.columns.length; j++) {
if (this.fieldsMetadata[i].column == this.dataset.meta.columns[j].column && this.dataset.meta.columns[j].pname == 'Type') {
this.dataset.meta.columns[j].pvalue = this.fieldsMetadata[i].Type
}
}
}
}
}
})
......
......@@ -44,7 +44,8 @@
<SparqlDataset v-if="dataset.dsTypeCd == 'SPARQL'" :selectedDataset="selectedDataset" />
<SolrDataset v-if="dataset.dsTypeCd == 'Solr'" :selectedDataset="selectedDataset" />
<PythonDataset v-if="dataset.dsTypeCd == 'Python/R'" :selectedDataset="selectedDataset" :pythonEnvironments="pythonEnvironments" :rEnvironments="rEnvironments" />
<ParamTable v-if="dataset.dsTypeCd && dataset.dsTypeCd != 'File' && dataset.dsTypeCd != 'Flat'" :selectedDataset="selectedDataset" />
<PreparedDataset v-if="dataset.dsTypeCd == 'Prepared'" :selectedDataset="selectedDataset" :pythonEnvironments="pythonEnvironments" :rEnvironments="rEnvironments" />
<ParamTable v-if="dataset.dsTypeCd && dataset.dsTypeCd != 'File' && dataset.dsTypeCd != 'Flat' && dataset.dsTypeCd != 'Prepared'" :selectedDataset="selectedDataset" />
</template>
<script lang="ts">
......@@ -67,8 +68,9 @@ import RestDataset from './restDataset/DatasetManagementRestDataset.vue'
import SparqlDataset from './sparqlDataset/DatasetManagementSparqlDataset.vue'
import SolrDataset from './solrDataset/DatasetManagementSolrDataset.vue'
import PythonDataset from './pythonDataset/DatasetManagementPythonDataset.vue'
import PreparedDataset from './preparedDataset/DatasetManagementPreparedDataset.vue'
export default defineComponent({
components: { Card, Dropdown, KnValidationMessages, ParamTable, CkanDataset, QbeDataset, RestDataset, JavaDataset, FlatDataset, SolrDataset, QueryDataset, ScriptDataset, SparqlDataset, PythonDataset, FileDataset },
components: { Card, Dropdown, KnValidationMessages, ParamTable, CkanDataset, QbeDataset, RestDataset, JavaDataset, FlatDataset, SolrDataset, QueryDataset, ScriptDataset, SparqlDataset, PythonDataset, FileDataset, PreparedDataset },
props: {
parentValid: { type: Boolean },
selectedDataset: { type: Object as any },
......
{
"validations": {
"dataset": [
{
"fieldName": "flatTableName",
"validators": [
{
"key": "required",
"validator": {
"type": "flat-fields-required"
}
}
]
},
{
"fieldName": "dataSourceFlat",
"validators": [
{
"key": "required",
"validator": {
"type": "flat-fields-required"
}
}
]
}
]
}
}
<template>
<Card class="p-m-2">
<template #content>
<form class="p-fluid p-formgrid p-grid">
<div class="p-field p-col-6">
<Button :label="$t('managers.datasetManagement.monitoring')" class="kn-button kn-button--primary" @click="showMonitoringDialog = true" />
</div>
<div class="p-field p-col-6">
<Button :label="$t('managers.datasetManagement.openDP')" class="kn-button kn-button--primary" @click="prepareForDataPreparation" />
</div>
</form>
</template>
</Card>
<MonitoringDialog :visibility="showMonitoringDialog" :dataset="selectedDataset" @close="showMonitoringDialog = false" />
</template>
<script lang="ts">
import { defineComponent } from 'vue'
import { AxiosResponse } from 'axios'
import descriptor from './DatasetManagementPreparedDataset.json'
import Card from 'primevue/card'
import MonitoringDialog from '@/modules/workspace/dataPreparation/DataPreparationMonitoring/DataPreparationMonitoringDialog.vue'
export default defineComponent({
components: { Card, MonitoringDialog },
props: { selectedDataset: { type: Object as any }, dataSources: { type: Array as any } },
emits: ['touched'],
data() {
return {
descriptor,
dataset: {} as any,
availableDatasets: [] as any,
avroDatasets: [] as any,
showMonitoringDialog: false
}
},
created() {
this.dataset = this.selectedDataset
},
watch: {
selectedDataset() {
this.dataset = this.selectedDataset
}
},
methods: {
routeToDataPreparation() {
let path = ''
this.$confirm.require({
header: this.$t('managers.datasetManagement.openDP'),
message: this.$t('managers.datasetManagement.confirmMsg'),
icon: 'pi pi-exclamation-triangle',
accept: () => {
this.$router.push(path)
}
})
},
async getAllAvroDataSets() {
await this.$http
.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `3.0/datasets/avro`)
.then((response: AxiosResponse<any>) => {
this.avroDatasets = response.data
})
.catch(() => {})
},
async prepareForDataPreparation() {
await this.getAllAvroDataSets()
await this.openDataPreparation(this.selectedDataset)
},
isAvroReady(dsLabel: String) {
if (this.avroDatasets.indexOf(dsLabel) >= 0) return true
else return false
},
openDataPreparation(dataset: any) {
if (dataset.dsTypeCd == 'Prepared') {
//edit existing data prep
this.$http.get(process.env.VUE_APP_RESTFUL_SERVICES_PATH + `3.0/datasets/advanced/${dataset.label}`).then(
(response: AxiosResponse<any>) => {
let instanceId = response.data.configuration.dataPrepInstanceId
this.$http.get(process.env.VUE_APP_DATA_PREPARATION_PATH + `1.0/process/by-instance-id/${instanceId}`).then(
(response: AxiosResponse<any>) => {
let transformations = response.data.definition
let processId = response.data.id
let datasetLabel = response.data.instance.dataSetLabel
if (this.isAvroReady(datasetLabel))
// check if Avro file has been deleted or not
this.$router.push({ name: 'data-preparation', params: { id: datasetLabel, transformations: JSON.stringify(transformations), processId: processId, instanceId: instanceId, dataset: JSON.stringify(dataset) } })
else {
this.$store.commit('setInfo', {
title: 'Avro file is missing',
msg: 'Generate it again and then retry'
})
}
},
() => {
this.$store.commit('setError', { title: 'Save error', msg: 'Cannot create process' })
}
)
},
() => {
this.$store.commit('setError', {
title: 'Cannot open data preparation'
})
}
)
} else if (this.isAvroReady(dataset.label)) {
// original dataset already exported in Avro
this.$router.push({ name: 'data-preparation', params: { id: dataset.label } })
} else {
this.$store.commit('setInfo', {
title: 'Avro file is missing',
msg: 'Generate it again and then retry'
})
}
}
}
})
</script>
......@@ -115,7 +115,7 @@
<QBEParamDialog v-if="paramDialogVisible" :visible="paramDialogVisible" :propDataset="qbe" @close="paramDialogVisible = false" @save="onParametersSave" />
<QBEHavingDialog :visible="havingDialogVisible" :havingDialogData="havingDialogData" :entities="selectedQuery?.fields" @close="havingDialogVisible = false" @save="onHavingsSave"></QBEHavingDialog>
<QBEAdvancedFilterDialog :visible="advancedFilterDialogVisible" :query="selectedQuery" @close="advancedFilterDialogVisible = false" @save="onAdvancedFiltersSave"></QBEAdvancedFilterDialog>
<QBESavingDialog v-if="savingDialogVisible" :visible="savingDialogVisible" :propDataset="qbe" @close="savingDialogVisible = false" @datasetSaved="$emit('datasetSaved')" />
<QBESavingDialog :visible="savingDialogVisible" :propDataset="qbe" @close="savingDialogVisible = false" @datasetSaved="$emit('datasetSaved')" />
<QBEJoinDefinitionDialog v-if="joinDefinitionDialogVisible" :visible="joinDefinitionDialogVisible" :qbe="qbe" :propEntities="entities?.entities" :id="uniqueID" :selectedQuery="selectedQuery" @close="onJoinDefinitionDialogClose"></QBEJoinDefinitionDialog>
<KnCalculatedField v-model:template="selectedCalcField" v-model:visibility="calcFieldDialogVisible" :fields="calcFieldColumns" :descriptor="calcFieldDescriptor" :readOnly="false" :valid="true" source="QBE" @save="onCalcFieldSave" @cancel="calcFieldDialogVisible = false">
......
......@@ -78,8 +78,11 @@ export default defineComponent({
this.selectedDataset = this.propDataset
},
watch: {
propDataset() {
this.selectedDataset = this.propDataset
propDataset: {
handler() {
this.selectedDataset = this.propDataset
},
deep: true
}
},
methods: {
......@@ -93,8 +96,6 @@ export default defineComponent({
},
async saveDataset() {
console.log('dataset', this.selectedDataset)
let dsToSave = { ...this.selectedDataset } as any
dsToSave.pars ? '' : (dsToSave.pars = [])
dsToSave.pythonEnvironment ? (dsToSave.pythonEnvironment = JSON.stringify(dsToSave.pythonEnvironment)) : ''
......@@ -136,7 +137,7 @@ export default defineComponent({
columnsNames = this.removeDuplicates(columnsNames)
for (i = 0; i < columnsNames.length; i++) {
var columnObject = { displayedName: '', name: '', fieldType: '', type: '' }
var columnObject = { displayedName: '', name: '', fieldType: '', type: '', personal: false, decript: false, subjectId: false }
var currentColumnName = columnsNames[i]
if (currentColumnName.indexOf(':') != -1) {
......@@ -154,6 +155,12 @@ export default defineComponent({
columnObject.type = element.pvalue
} else if (element.pname.toUpperCase() == 'fieldType'.toUpperCase()) {
columnObject.fieldType = element.pvalue
} else if (element.pname.toUpperCase() == 'personal'.toUpperCase()) {
columnObject.personal = element.pvalue
} else if (element.pname.toUpperCase() == 'decript'.toUpperCase()) {
columnObject.decript = element.pvalue
} else if (element.pname.toUpperCase() == 'subjectId'.toUpperCase()) {
columnObject.subjectId = element.pvalue
}
}
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment