From eef5936282bf2e9bae7767409b78a27954aa2e2e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 10 May 2023 17:04:57 -0500 Subject: [PATCH] bulk insert & delete w/ pre-validation and warnings and errors and such --- .../AbstractPreDeleteCustomizer.java | 44 +++- .../AbstractPreInsertCustomizer.java | 39 ++++ .../AbstractPreUpdateCustomizer.java | 5 + .../core/actions/tables/DeleteAction.java | 219 ++++++++++-------- .../core/actions/tables/InsertAction.java | 41 ++-- .../core/instances/QInstanceEnricher.java | 4 +- .../bulk/delete/BulkDeleteLoadStep.java | 156 +++++++++++++ .../bulk/delete/BulkDeleteTransformStep.java | 85 +++++-- .../bulk/edit/BulkEditLoadStep.java | 13 +- .../bulk/edit/BulkEditTransformStep.java | 35 +-- .../bulk/insert/BulkInsertTransformStep.java | 88 +++++-- .../LoadViaDeleteStep.java | 5 +- ...ProcessSummaryWarningsAndErrorsRollup.java | 40 +++- 13 files changed, 583 insertions(+), 191 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java index de03076f..4b848a14 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.customizers; import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -31,6 +32,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** Abstract class that a table can specify an implementation of, to provide ** custom actions before a delete takes place. ** + ** It's important for implementations to be aware of the isPreview field, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** ** General implementation would be, to iterate over the records (which the DeleteAction ** would look up based on the inputs to the delete action), and look at their values: ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records @@ -43,20 +49,19 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** ** Note that the full deleteInput is available as a field in this class. ** - ** A future enhancement here may be to take (as fields in this class) the list of - ** records that the delete action marked in error - the user might want to do - ** something special with them (idk, try some other way to delete them?) *******************************************************************************/ public abstract class AbstractPreDeleteCustomizer { protected DeleteInput deleteInput; + protected boolean isPreview = false; + /******************************************************************************* ** *******************************************************************************/ - public abstract List apply(List records); + public abstract List apply(List records) throws QException; @@ -80,4 +85,35 @@ public abstract class AbstractPreDeleteCustomizer this.deleteInput = deleteInput; } + + + /******************************************************************************* + ** Getter for isPreview + *******************************************************************************/ + public boolean getIsPreview() + { + return (this.isPreview); + } + + + + /******************************************************************************* + ** Setter for isPreview + *******************************************************************************/ + public void setIsPreview(boolean isPreview) + { + this.isPreview = isPreview; + } + + + + /******************************************************************************* + ** Fluent setter for isPreview + *******************************************************************************/ + public AbstractPreDeleteCustomizer withIsPreview(boolean isPreview) + { + this.isPreview = isPreview; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java index 5704cdc9..d1e21dc4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java @@ -32,6 +32,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** Abstract class that a table can specify an implementation of, to provide ** custom actions before an insert takes place. ** + ** It's important for implementations to be aware of the isPreview field, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** ** General implementation would be, to iterate over the records (the inputs to ** the insert action), and look at their values: ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records @@ -46,6 +51,8 @@ public abstract class AbstractPreInsertCustomizer { protected InsertInput insertInput; + protected boolean isPreview = false; + /******************************************************************************* @@ -74,4 +81,36 @@ public abstract class AbstractPreInsertCustomizer { this.insertInput = insertInput; } + + + + /******************************************************************************* + ** Getter for isPreview + *******************************************************************************/ + public boolean getIsPreview() + { + return (this.isPreview); + } + + + + /******************************************************************************* + ** Setter for isPreview + *******************************************************************************/ + public void setIsPreview(boolean isPreview) + { + this.isPreview = isPreview; + } + + + + /******************************************************************************* + ** Fluent setter for isPreview + *******************************************************************************/ + public AbstractPreInsertCustomizer withIsPreview(boolean isPreview) + { + this.isPreview = isPreview; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java index 3a1a13da..b8a95ed2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java @@ -35,6 +35,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** Abstract class that a table can specify an implementation of, to provide ** custom actions before an update takes place. ** + ** It's important for implementations to be aware of the isPreview field, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** ** General implementation would be, to iterate over the records (the inputs to ** the update action), and look at their values: ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index f8138030..32e0320a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -26,8 +26,10 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -79,14 +81,32 @@ public class DeleteAction { ActionHelper.validateSession(deleteInput); - QTableMetaData table = deleteInput.getTable(); - String primaryKeyField = table.getPrimaryKeyField(); + QTableMetaData table = deleteInput.getTable(); + String primaryKeyFieldName = table.getPrimaryKeyField(); + QFieldMetaData primaryKeyField = table.getField(primaryKeyFieldName); - if(CollectionUtils.nullSafeHasContents(deleteInput.getPrimaryKeys()) && deleteInput.getQueryFilter() != null) + List primaryKeys = deleteInput.getPrimaryKeys(); + if(CollectionUtils.nullSafeHasContents(primaryKeys) && deleteInput.getQueryFilter() != null) { throw (new QException("A delete request may not contain both a list of primary keys and a query filter.")); } + //////////////////////////////////////////////////////// + // make sure the primary keys are of the correct type // + //////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(primaryKeys)) + { + for(int i = 0; i < primaryKeys.size(); i++) + { + Serializable primaryKey = primaryKeys.get(i); + Serializable valueAsFieldType = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKey); + if(!Objects.equals(primaryKey, valueAsFieldType)) + { + primaryKeys.set(i, valueAsFieldType); + } + } + } + ////////////////////////////////////////////////////// // load the backend module and its delete interface // ////////////////////////////////////////////////////// @@ -119,50 +139,32 @@ public class DeleteAction //////////////////////////////////////////////////////////////////////////////// Optional> oldRecordList = fetchOldRecords(deleteInput, deleteInterface); - List recordsWithValidationErrors = new ArrayList<>(); - List recordsWithValidationWarnings = new ArrayList<>(); - if(oldRecordList.isPresent()) + List customizerResult = performValidations(deleteInput, oldRecordList, false); + List recordsWithValidationErrors = new ArrayList<>(); + Map recordsWithValidationWarnings = new LinkedHashMap<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // check if any records got errors in the customizer - if so, remove them from the input list of pkeys to delete // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(customizerResult != null) { - recordsWithValidationErrors = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get()); - } - - /////////////////////////////////////////////////////////////////////////// - // after all validations, run the pre-delete customizer, if there is one // - /////////////////////////////////////////////////////////////////////////// - Optional preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole()); - if(preDeleteCustomizer.isPresent() && oldRecordList.isPresent()) - { - //////////////////////////////////////////////////////////////////////////// - // make list of records that are still good - to pass into the customizer // - //////////////////////////////////////////////////////////////////////////// - List recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyField, oldRecordList.get(), recordsWithValidationErrors); - - preDeleteCustomizer.get().setDeleteInput(deleteInput); - List customizerResult = preDeleteCustomizer.get().apply(recordsForCustomizer); - - /////////////////////////////////////////////////////// - // check if any records got errors in the customizer // - /////////////////////////////////////////////////////// Set primaryKeysToRemoveFromInput = new HashSet<>(); for(QRecord record : customizerResult) { if(CollectionUtils.nullSafeHasContents(record.getErrors())) { recordsWithValidationErrors.add(record); - primaryKeysToRemoveFromInput.add(record.getValue(primaryKeyField)); + primaryKeysToRemoveFromInput.add(record.getValue(primaryKeyFieldName)); } else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) { - recordsWithValidationWarnings.add(record); + recordsWithValidationWarnings.put(record.getValue(primaryKeyFieldName), record); } } - ///////////////////////////////////////////////////////////////// - // do one mass removal of any bad keys from the input key list // - ///////////////////////////////////////////////////////////////// if(!primaryKeysToRemoveFromInput.isEmpty()) { - deleteInput.getPrimaryKeys().removeAll(primaryKeysToRemoveFromInput); + primaryKeys.removeAll(primaryKeysToRemoveFromInput); } } @@ -171,24 +173,34 @@ public class DeleteAction //////////////////////////////////// DeleteOutput deleteOutput = deleteInterface.execute(deleteInput); - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // merge the backend's output with any validation errors we found (whose ids wouldn't have gotten into the backend delete) // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List outputRecordsWithErrors = deleteOutput.getRecordsWithErrors(); - if(outputRecordsWithErrors == null) - { - deleteOutput.setRecordsWithErrors(new ArrayList<>()); - outputRecordsWithErrors = deleteOutput.getRecordsWithErrors(); - } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // merge the backend's output with any validation errors we found (whose pkeys wouldn't have gotten into the backend delete) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List outputRecordsWithErrors = Objects.requireNonNullElseGet(deleteOutput.getRecordsWithErrors(), () -> new ArrayList<>()); outputRecordsWithErrors.addAll(recordsWithValidationErrors); - List outputRecordsWithWarnings = deleteOutput.getRecordsWithWarnings(); - if(outputRecordsWithWarnings == null) + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if a record had a validation warning, but then an execution error, remove it from the warning list - so it's only in one of them. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord outputRecordWithError : outputRecordsWithErrors) { - deleteOutput.setRecordsWithWarnings(new ArrayList<>()); - outputRecordsWithWarnings = deleteOutput.getRecordsWithWarnings(); + Serializable pkey = outputRecordWithError.getValue(primaryKeyFieldName); + recordsWithValidationWarnings.remove(pkey); + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // combine the warning list from validation to that from execution - avoiding duplicates // + // use a map to manage this list for the rest of this method // + /////////////////////////////////////////////////////////////////////////////////////////// + Map outputRecordsWithWarningMap = CollectionUtils.nullSafeIsEmpty(deleteOutput.getRecordsWithWarnings()) ? new LinkedHashMap<>() + : deleteOutput.getRecordsWithWarnings().stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyFieldName), r -> r, (a, b) -> a, () -> new LinkedHashMap<>())); + for(Map.Entry entry : recordsWithValidationWarnings.entrySet()) + { + if(!outputRecordsWithWarningMap.containsKey(entry.getKey())) + { + outputRecordsWithWarningMap.put(entry.getKey(), entry.getValue()); + } } - outputRecordsWithWarnings.addAll(recordsWithValidationWarnings); //////////////////////////////////////// // delete associations, if applicable // @@ -212,25 +224,27 @@ public class DeleteAction //////////////////////////////////////////////////////////////////////////// // make list of records that are still good - to pass into the customizer // //////////////////////////////////////////////////////////////////////////// - List recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyField, oldRecordList.get(), outputRecordsWithErrors); + List recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyFieldName, oldRecordList.get(), outputRecordsWithErrors); try { postDeleteCustomizer.get().setDeleteInput(deleteInput); - List customizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer); + List postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer); /////////////////////////////////////////////////////// // check if any records got errors in the customizer // /////////////////////////////////////////////////////// - for(QRecord record : customizerResult) + for(QRecord record : postCustomizerResult) { + Serializable pkey = record.getValue(primaryKeyFieldName); if(CollectionUtils.nullSafeHasContents(record.getErrors())) { outputRecordsWithErrors.add(record); + outputRecordsWithWarningMap.remove(pkey); } else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) { - outputRecordsWithWarnings.add(record); + outputRecordsWithWarningMap.put(pkey, record); } } } @@ -239,16 +253,65 @@ public class DeleteAction for(QRecord record : recordsForCustomizer) { record.addWarning(new QWarningMessage("An error occurred after the delete: " + e.getMessage())); - outputRecordsWithWarnings.add(record); + outputRecordsWithWarningMap.put(record.getValue(primaryKeyFieldName), record); } } } + deleteOutput.setRecordsWithErrors(outputRecordsWithErrors); + deleteOutput.setRecordsWithWarnings(new ArrayList<>(outputRecordsWithWarningMap.values())); + return deleteOutput; } + /******************************************************************************* + ** this method takes in the deleteInput, and the list of old records that matched + ** the pkeys in that input. + ** + ** it'll check if any of those pkeys aren't found (in a sub-method) - a record + ** with an error message will be added to oldRecordList for any such records. + ** + ** it'll also then call the pre-customizer, if there is one - taking in the + ** oldRecordList. it can add other errors or warnings to records. + ** + ** The return value here is basically oldRecordList - possibly with some new + ** entries for the pkey-not-founds, and possibly w/ errors and warnings from the + ** customizer. + *******************************************************************************/ + public List performValidations(DeleteInput deleteInput, Optional> oldRecordList, boolean isPreview) throws QException + { + if(oldRecordList.isEmpty()) + { + return (null); + } + + QTableMetaData table = deleteInput.getTable(); + List primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get()); + + /////////////////////////////////////////////////////////////////////////// + // after all validations, run the pre-delete customizer, if there is one // + /////////////////////////////////////////////////////////////////////////// + Optional preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole()); + List customizerResult = oldRecordList.get(); + if(preDeleteCustomizer.isPresent()) + { + preDeleteCustomizer.get().setDeleteInput(deleteInput); + preDeleteCustomizer.get().setIsPreview(isPreview); + customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get()); + } + + ///////////////////////////////////////////////////////////////////////// + // add any pkey-not-found records to the front of the customizerResult // + ///////////////////////////////////////////////////////////////////////// + customizerResult.addAll(primaryKeysNotFound); + + return customizerResult; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -344,9 +407,9 @@ public class DeleteAction ** records that you can't see because of security - that they won't be found ** by the query here, so it's the same to you as if they don't exist at all! ** - ** This method, if it finds any missing records, will: - ** - remove those ids from the deleteInput - ** - create a QRecord with that id and a not-found error message. + ** If this method identifies any missing records (e.g., from PKeys that are + ** requested to be deleted, but don't exist (or can't be seen)), then it will + ** return those as new QRecords, with error messages. *******************************************************************************/ private List validateRecordsExistAndCanBeAccessed(DeleteInput deleteInput, List oldRecordList) throws QException { @@ -355,64 +418,28 @@ public class DeleteAction QTableMetaData table = deleteInput.getTable(); QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField()); - Set primaryKeysToRemoveFromInput = new HashSet<>(); - List> pages = CollectionUtils.getPages(deleteInput.getPrimaryKeys(), 1000); for(List page : pages) { - List primaryKeysToLookup = new ArrayList<>(); - for(Serializable primaryKeyValue : page) + Map oldRecordMapByPrimaryKey = new HashMap<>(); + for(QRecord record : oldRecordList) { - if(primaryKeyValue != null) - { - primaryKeysToLookup.add(primaryKeyValue); - } - } - - Map lookedUpRecords = new HashMap<>(); - if(CollectionUtils.nullSafeHasContents(oldRecordList)) - { - for(QRecord record : oldRecordList) - { - Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField()); - primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue); - lookedUpRecords.put(primaryKeyValue, record); - } - } - else if(!primaryKeysToLookup.isEmpty()) - { - QueryInput queryInput = new QueryInput(); - queryInput.setTransaction(deleteInput.getTransaction()); - queryInput.setTableName(table.getName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeysToLookup))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - for(QRecord record : queryOutput.getRecords()) - { - lookedUpRecords.put(record.getValue(table.getPrimaryKeyField()), record); - } + Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField()); + primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue); + oldRecordMapByPrimaryKey.put(primaryKeyValue, record); } for(Serializable primaryKeyValue : page) { primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue); - if(!lookedUpRecords.containsKey(primaryKeyValue)) + if(!oldRecordMapByPrimaryKey.containsKey(primaryKeyValue)) { QRecord recordWithError = new QRecord(); recordsWithErrors.add(recordWithError); recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue); recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue)); - primaryKeysToRemoveFromInput.add(primaryKeyValue); } } - - ///////////////////////////////////////////////////////////////// - // do one mass removal of any bad keys from the input key list // - ///////////////////////////////////////////////////////////////// - if(!primaryKeysToRemoveFromInput.isEmpty()) - { - deleteInput.getPrimaryKeys().removeAll(primaryKeysToRemoveFromInput); - primaryKeysToRemoveFromInput.clear(); - } } return (recordsWithErrors); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index eebdc8c4..7198ae9f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -101,20 +101,7 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); - if(preInsertCustomizer.isPresent()) - { - preInsertCustomizer.get().setInsertInput(insertInput); - insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords())); - } + performValidations(insertInput, false); //////////////////////////////////// // have the backend do the insert // @@ -172,6 +159,32 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + if(preInsertCustomizer.isPresent()) + { + preInsertCustomizer.get().setInsertInput(insertInput); + preInsertCustomizer.get().setIsPreview(isPreview); + insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords())); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 4b1bfa9f..c8ea8a60 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -65,13 +65,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QMiddlewareTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -808,7 +808,7 @@ public class QInstanceEnricher QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( ExtractViaQueryStep.class, BulkDeleteTransformStep.class, - LoadViaDeleteStep.class, + BulkDeleteLoadStep.class, values ) .withName(processName) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java new file mode 100644 index 00000000..75e4db9b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java @@ -0,0 +1,156 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Generic implementation of a LoadStep - that runs a Delete action for a + ** specified table. + *******************************************************************************/ +public class BulkDeleteLoadStep extends LoadViaDeleteStep implements ProcessSummaryProviderInterface +{ + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + private List infoSummaries = new ArrayList<>(); + + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("deleted"); + + private String tableLabel; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + super.preRun(runBackendStepInput, runBackendStepOutput); + + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + if(table != null) + { + tableLabel = table.getLabel(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + ArrayList rs = new ArrayList<>(); + + String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; + + okSummary.setSingularPastMessage(tableLabel + " record was deleted" + noWarningsSuffix + "."); + okSummary.setPluralPastMessage(tableLabel + " records were deleted" + noWarningsSuffix + "."); + okSummary.pickMessage(isForResultScreen); + okSummary.addSelfToListIfAnyCount(rs); + + processSummaryWarningsAndErrorsRollup.addToList(rs); + rs.addAll(infoSummaries); + return (rs); + } + + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + //////////////////////////// + // have base class delete // + //////////////////////////// + super.run(runBackendStepInput, runBackendStepOutput); + + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + String primaryKeyFieldName = table.getPrimaryKeyField(); + Map outputRecordMap = runBackendStepOutput.getRecords().stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyFieldName), r -> r, (a, b) -> a)); + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // roll up the results, based on the input list, but looking for error/warnings from output list // + /////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : runBackendStepInput.getRecords()) + { + Serializable recordPrimaryKey = record.getValue(primaryKeyFieldName); + QRecord outputRecord = outputRecordMap.get(recordPrimaryKey); + + if(CollectionUtils.nullSafeHasContents(outputRecord.getErrors())) + { + String message = outputRecord.getErrors().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addError(message, recordPrimaryKey); + } + else if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) + { + String message = outputRecord.getWarnings().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addWarning(message, recordPrimaryKey); + } + else + { + okSummary.incrementCountAndAddPrimaryKey(recordPrimaryKey); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Optional openTransaction(RunBackendStepInput runBackendStepInput) throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); + + return (Optional.of(new InsertAction().openTransaction(insertInput))); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java index 6d0b353e..d3eac18f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java @@ -22,16 +22,24 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; +import java.io.Serializable; import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* @@ -41,6 +49,8 @@ public class BulkDeleteTransformStep extends AbstractTransformStep { private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("deleted"); + private String tableLabel; @@ -69,11 +79,15 @@ public class BulkDeleteTransformStep extends AbstractTransformStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + QTableMetaData table = runBackendStepInput.getTable(); + String primaryKeyField = table.getPrimaryKeyField(); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) + if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) || + runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_PREVIEW)) { if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null) { @@ -83,6 +97,42 @@ public class BulkDeleteTransformStep extends AbstractTransformStep { runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record"); } + + /////////////////////////////////////////////////////////////////////// + // run the validation - critically - in preview mode (boolean param) // + /////////////////////////////////////////////////////////////////////// + DeleteAction deleteAction = new DeleteAction(); + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(runBackendStepInput.getTableName()); + deleteInput.setPrimaryKeys(runBackendStepInput.getRecords().stream().map(r -> r.getValue(primaryKeyField)).toList()); + List validationResultRecords = deleteAction.performValidations(deleteInput, Optional.of(runBackendStepInput.getRecords()), true); + + ///////////////////////////////////////////////////////////// + // look at the update input to build process summary lines // + ///////////////////////////////////////////////////////////// + List outputRecords = new ArrayList<>(); + for(QRecord record : validationResultRecords) + { + Serializable recordPrimaryKey = record.getValue(table.getPrimaryKeyField()); + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + String message = record.getErrors().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addError(message, recordPrimaryKey); + } + else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) + { + String message = record.getWarnings().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addWarning(message, recordPrimaryKey); + outputRecords.add(record); + } + else + { + okSummary.incrementCountAndAddPrimaryKey(recordPrimaryKey); + outputRecords.add(record); + } + } + + runBackendStepOutput.setRecords(outputRecords); } else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE)) { @@ -94,13 +144,15 @@ public class BulkDeleteTransformStep extends AbstractTransformStep { runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " records"); } - } - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - // no transformation needs done - just pass records through from input to output, and assume all are OK // - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); - okSummary.incrementCount(runBackendStepInput.getRecords().size()); + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // no transformation needs done - just pass records through from input to output, and assume errors & warnings will come from the delete action // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); + + // i think load step will do this. + // okSummary.incrementCount(runBackendStepInput.getRecords().size()); + } } @@ -111,17 +163,16 @@ public class BulkDeleteTransformStep extends AbstractTransformStep @Override public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) { - if(isForResultScreen) - { - okSummary.setMessage(tableLabel + " records were deleted."); - } - else - { - okSummary.setMessage(tableLabel + " records will be deleted."); - } - ArrayList rs = new ArrayList<>(); - rs.add(okSummary); + + String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; + okSummary.setSingularFutureMessage(tableLabel + " record will be deleted" + noWarningsSuffix + "."); + okSummary.setPluralFutureMessage(tableLabel + " records will be deleted" + noWarningsSuffix + "."); + okSummary.pickMessage(isForResultScreen); + okSummary.addSelfToListIfAnyCount(rs); + + processSummaryWarningsAndErrorsRollup.addToList(rs); + return (rs); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java index d281d3f7..561dee43 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java @@ -38,7 +38,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.buildInfoSummaryLines; -import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.getProcessSummaryWarningsAndErrorsRollup; /******************************************************************************* @@ -51,7 +50,7 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); private List infoSummaries = new ArrayList<>(); - private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = getProcessSummaryWarningsAndErrorsRollup(); + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited"); private String tableLabel; @@ -104,9 +103,15 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); - + //////////////////////////// + // have base class update // + //////////////////////////// super.run(runBackendStepInput, runBackendStepOutput); + + //////////////////////////////////////////////////////// + // roll up results based on output from update action // + //////////////////////////////////////////////////////// + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); for(QRecord record : runBackendStepOutput.getRecords()) { Serializable recordPrimaryKey = record.getValue(table.getPrimaryKeyField()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java index ddb8d2c6..8fb095e3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java @@ -59,7 +59,7 @@ public class BulkEditTransformStep extends AbstractTransformStep private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); private List infoSummaries = new ArrayList<>(); - private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = getProcessSummaryWarningsAndErrorsRollup(); + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited"); private QTableMetaData table; private String tableLabel; @@ -71,36 +71,6 @@ public class BulkEditTransformStep extends AbstractTransformStep - /******************************************************************************* - ** used by Load step too - *******************************************************************************/ - static ProcessSummaryWarningsAndErrorsRollup getProcessSummaryWarningsAndErrorsRollup() - { - return new ProcessSummaryWarningsAndErrorsRollup() - .withErrorTemplate(new ProcessSummaryLine(Status.ERROR) - .withSingularFutureMessage("record has an error: ") - .withPluralFutureMessage("records have an error: ") - .withSingularPastMessage("record had an error: ") - .withPluralPastMessage("records had an error: ")) - .withWarningTemplate(new ProcessSummaryLine(Status.WARNING) - .withSingularFutureMessage("record will be edited, but has a warning: ") - .withPluralFutureMessage("records will be edited, but have a warning: ") - .withSingularPastMessage("record was edited, but had a warning: ") - .withPluralPastMessage("records were edited, but had a warning: ")) - .withOtherErrorsSummary(new ProcessSummaryLine(Status.ERROR) - .withSingularFutureMessage("record has an other error.") - .withPluralFutureMessage("records have other errors.") - .withSingularPastMessage("record had an other error.") - .withPluralPastMessage("records had other errors.")) - .withOtherWarningsSummary(new ProcessSummaryLine(Status.WARNING) - .withSingularFutureMessage("record will be edited, but has an other warning.") - .withPluralFutureMessage("records will be edited, but have other warnings.") - .withSingularPastMessage("record was edited, but had other warnings.") - .withPluralPastMessage("records were edited, but had other warnings.")); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -169,7 +139,8 @@ public class BulkEditTransformStep extends AbstractTransformStep setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, recordToUpdate); } - okSummary.incrementCount(runBackendStepInput.getRecords().size()); + // i think load step will do this. + // okSummary.incrementCount(runBackendStepInput.getRecords().size()); } else { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 5252c954..b775098d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; @@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -51,7 +54,10 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; *******************************************************************************/ public class BulkInsertTransformStep extends AbstractTransformStep { - private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); + private Map ukErrorSummaries = new HashMap<>(); private QTableMetaData table; @@ -112,20 +118,24 @@ public class BulkInsertTransformStep extends AbstractTransformStep } } - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // no transformation needs to be done - just pass records through from input to output, if they don't violate any UK's // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction // + // will only be getting the records in pages, but in here, we'll track UK's across pages!! // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - /////////////////////////////////////////////////// - // if there are no UK's, just output all records // - /////////////////////////////////////////////////// + //////////////////////////////////////////////////// + // if there are no UK's, proceed with all records // + //////////////////////////////////////////////////// + List recordsWithoutUkErrors = new ArrayList<>(); if(existingKeys.isEmpty()) { - runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); - okSummary.incrementCount(runBackendStepInput.getRecords().size()); + recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords()); } else { + ///////////////////////////////////////////////////////////// + // else, only proceed with records that don't violate a UK // + ///////////////////////////////////////////////////////////// for(UniqueKey uniqueKey : uniqueKeys) { keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>()); @@ -162,11 +172,47 @@ public class BulkInsertTransformStep extends AbstractTransformStep Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv)); } - okSummary.incrementCount(); - runBackendStepOutput.addRecord(record); + recordsWithoutUkErrors.add(record); } } } + + ///////////////////////////////////////////////////////////////////////////////// + // run all validation from the insert action - in Preview mode (boolean param) // + ///////////////////////////////////////////////////////////////////////////////// + InsertAction insertAction = new InsertAction(); + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(runBackendStepInput.getTableName()); + insertInput.setRecords(recordsWithoutUkErrors); + insertInput.setSkipUniqueKeyCheck(true); + insertAction.performValidations(insertInput, true); + List validationResultRecords = insertInput.getRecords(); + + ///////////////////////////////////////////////////////////////// + // look at validation results to build process summary results // + ///////////////////////////////////////////////////////////////// + List outputRecords = new ArrayList<>(); + for(QRecord record : validationResultRecords) + { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + String message = record.getErrors().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addError(message, null); + } + else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) + { + String message = record.getWarnings().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addWarning(message, null); + outputRecords.add(record); + } + else + { + okSummary.incrementCountAndAddPrimaryKey(null); + outputRecords.add(record); + } + } + + runBackendStepOutput.setRecords(outputRecords); } @@ -177,22 +223,22 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) { - String tableLabel = table == null ? "" : table.getLabel(); + ArrayList rs = new ArrayList<>(); + String tableLabel = table == null ? "" : table.getLabel(); - okSummary - .withSingularFutureMessage(tableLabel + " record will be inserted") - .withPluralFutureMessage(tableLabel + " records will be inserted") - .withSingularPastMessage(tableLabel + " record was inserted") - .withPluralPastMessage(tableLabel + " records were inserted"); - - ArrayList rs = new ArrayList<>(); + String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; + okSummary.setSingularFutureMessage(tableLabel + " record will be inserted" + noWarningsSuffix + "."); + okSummary.setPluralFutureMessage(tableLabel + " records will be inserted" + noWarningsSuffix + "."); + okSummary.setSingularPastMessage(tableLabel + " record was inserted" + noWarningsSuffix + "."); + okSummary.setPluralPastMessage(tableLabel + " records were inserted" + noWarningsSuffix + "."); + okSummary.pickMessage(isForResultScreen); okSummary.addSelfToListIfAnyCount(rs); for(Map.Entry entry : ukErrorSummaries.entrySet()) { UniqueKey uniqueKey = entry.getKey(); ProcessSummaryLine ukErrorSummary = entry.getValue(); - String ukErrorSuffix = " inserted, because they contain a duplicate key (" + uniqueKey.getDescription(table) + ")"; + String ukErrorSuffix = " inserted, because of duplicate values in a unique key (" + uniqueKey.getDescription(table) + ")"; ukErrorSummary .withSingularFutureMessage(tableLabel + " record will not be" + ukErrorSuffix) @@ -203,6 +249,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary.addSelfToListIfAnyCount(rs); } + processSummaryWarningsAndErrorsRollup.addToList(rs); + return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java index 3d67a810..fa0ebb9a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -60,7 +61,9 @@ public class LoadViaDeleteStep extends AbstractLoadStep deleteInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); // todo? can make more efficient deletes, maybe? deleteInput.setQueryFilter(); getTransaction().ifPresent(deleteInput::setTransaction); - new DeleteAction().execute(deleteInput); + DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput); + runBackendStepOutput.getRecords().addAll(deleteOutput.getRecordsWithErrors()); + runBackendStepOutput.getRecords().addAll(deleteOutput.getRecordsWithWarnings()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java index 8bef8eeb..91f343bd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java @@ -48,6 +48,36 @@ public class ProcessSummaryWarningsAndErrorsRollup + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessSummaryWarningsAndErrorsRollup build(String pastTenseVerb) + { + return new ProcessSummaryWarningsAndErrorsRollup() + .withErrorTemplate(new ProcessSummaryLine(Status.ERROR) + .withSingularFutureMessage("record has an error: ") + .withPluralFutureMessage("records have an error: ") + .withSingularPastMessage("record had an error: ") + .withPluralPastMessage("records had an error: ")) + .withWarningTemplate(new ProcessSummaryLine(Status.WARNING) + .withSingularFutureMessage("record will be " + pastTenseVerb + ", but has a warning: ") + .withPluralFutureMessage("records will be " + pastTenseVerb + ", but have a warning: ") + .withSingularPastMessage("record was " + pastTenseVerb + ", but had a warning: ") + .withPluralPastMessage("records were " + pastTenseVerb + ", but had a warning: ")) + .withOtherErrorsSummary(new ProcessSummaryLine(Status.ERROR) + .withSingularFutureMessage("record has an other error.") + .withPluralFutureMessage("records have other errors.") + .withSingularPastMessage("record had an other error.") + .withPluralPastMessage("records had other errors.")) + .withOtherWarningsSummary(new ProcessSummaryLine(Status.WARNING) + .withSingularFutureMessage("record will be " + pastTenseVerb + ", but has an other warning.") + .withPluralFutureMessage("records will be " + pastTenseVerb + ", but have other warnings.") + .withSingularPastMessage("record was " + pastTenseVerb + ", but had other warnings.") + .withPluralPastMessage("records were " + pastTenseVerb + ", but had other warnings.")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -164,7 +194,15 @@ public class ProcessSummaryWarningsAndErrorsRollup } } } - processSummaryLine.incrementCountAndAddPrimaryKey(primaryKey); + + if(primaryKey == null) + { + processSummaryLine.incrementCount(); + } + else + { + processSummaryLine.incrementCountAndAddPrimaryKey(primaryKey); + } }