QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

bulk insert & delete w/ pre-validation and warnings and errors and such

Browse Source
This commit is contained in:
darin.kelkhoff
2023-05-10 17:04:57 -05:00
parent b9ad0e7e21
commit eef5936282
13 changed files with 583 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -31,6 +32,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
** Abstract class that a table can specify an implementation of, to provide
** custom actions before a delete takes place.
**
** It's important for implementations to be aware of the isPreview field, which
** is set to true when the code is running to give users advice, e.g., on a review
** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing
** things like storing data, you don't want to do that if isPreview is true!!
**
** General implementation would be, to iterate over the records (which the DeleteAction
** would look up based on the inputs to the delete action), and look at their values:
** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
@ -43,20 +49,19 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
**
** Note that the full deleteInput is available as a field in this class.
**
** A future enhancement here may be to take (as fields in this class) the list of
** records that the delete action marked in error - the user might want to do
** something special with them (idk, try some other way to delete them?)
*******************************************************************************/
public abstract class AbstractPreDeleteCustomizer
{
protected DeleteInput deleteInput;
protected boolean isPreview = false;
/*******************************************************************************
**
*******************************************************************************/
public abstract List<QRecord> apply(List<QRecord> records);
public abstract List<QRecord> apply(List<QRecord> records) throws QException;
@ -80,4 +85,35 @@ public abstract class AbstractPreDeleteCustomizer
this.deleteInput = deleteInput;
}
/*******************************************************************************
** Getter for isPreview
*******************************************************************************/
public boolean getIsPreview()
{
return (this.isPreview);
}
/*******************************************************************************
** Setter for isPreview
*******************************************************************************/
public void setIsPreview(boolean isPreview)
{
this.isPreview = isPreview;
}
/*******************************************************************************
** Fluent setter for isPreview
*******************************************************************************/
public AbstractPreDeleteCustomizer withIsPreview(boolean isPreview)
{
this.isPreview = isPreview;
return (this);
}
}

39
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java
View File

@ -32,6 +32,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
** Abstract class that a table can specify an implementation of, to provide
** custom actions before an insert takes place.
**
** It's important for implementations to be aware of the isPreview field, which
** is set to true when the code is running to give users advice, e.g., on a review
** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing
** things like storing data, you don't want to do that if isPreview is true!!
**
** General implementation would be, to iterate over the records (the inputs to
** the insert action), and look at their values:
** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
@ -46,6 +51,8 @@ public abstract class AbstractPreInsertCustomizer
{
protected InsertInput insertInput;
protected boolean isPreview = false;
/*******************************************************************************
@ -74,4 +81,36 @@ public abstract class AbstractPreInsertCustomizer
{
this.insertInput = insertInput;
}
/*******************************************************************************
** Getter for isPreview
*******************************************************************************/
public boolean getIsPreview()
{
return (this.isPreview);
}
/*******************************************************************************
** Setter for isPreview
*******************************************************************************/
public void setIsPreview(boolean isPreview)
{
this.isPreview = isPreview;
}
/*******************************************************************************
** Fluent setter for isPreview
*******************************************************************************/
public AbstractPreInsertCustomizer withIsPreview(boolean isPreview)
{
this.isPreview = isPreview;
return (this);
}
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java
View File

@ -35,6 +35,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
** Abstract class that a table can specify an implementation of, to provide
** custom actions before an update takes place.
**
** It's important for implementations to be aware of the isPreview field, which
** is set to true when the code is running to give users advice, e.g., on a review
** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing
** things like storing data, you don't want to do that if isPreview is true!!
**
** General implementation would be, to iterate over the records (the inputs to
** the update action), and look at their values:
** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records

209
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java
View File

@ -26,8 +26,10 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -80,13 +82,31 @@ public class DeleteAction
ActionHelper.validateSession(deleteInput);
QTableMetaData table = deleteInput.getTable();
String primaryKeyField = table.getPrimaryKeyField();
String primaryKeyFieldName = table.getPrimaryKeyField();
QFieldMetaData primaryKeyField = table.getField(primaryKeyFieldName);
if(CollectionUtils.nullSafeHasContents(deleteInput.getPrimaryKeys()) && deleteInput.getQueryFilter() != null)
List<Serializable> primaryKeys = deleteInput.getPrimaryKeys();
if(CollectionUtils.nullSafeHasContents(primaryKeys) && deleteInput.getQueryFilter() != null)
{
throw (new QException("A delete request may not contain both a list of primary keys and a query filter."));
}
////////////////////////////////////////////////////////
// make sure the primary keys are of the correct type //
////////////////////////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(primaryKeys))
{
for(int i = 0; i < primaryKeys.size(); i++)
{
Serializable primaryKey = primaryKeys.get(i);
Serializable valueAsFieldType = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKey);
if(!Objects.equals(primaryKey, valueAsFieldType))
{
primaryKeys.set(i, valueAsFieldType);
}
}
}
//////////////////////////////////////////////////////
// load the backend module and its delete interface //
//////////////////////////////////////////////////////
@ -119,50 +139,32 @@ public class DeleteAction
////////////////////////////////////////////////////////////////////////////////
Optional<List<QRecord>> oldRecordList = fetchOldRecords(deleteInput, deleteInterface);
List<QRecord> customizerResult = performValidations(deleteInput, oldRecordList, false);
List<QRecord> recordsWithValidationErrors = new ArrayList<>();
List<QRecord> recordsWithValidationWarnings = new ArrayList<>();
if(oldRecordList.isPresent())
Map<Serializable, QRecord> recordsWithValidationWarnings = new LinkedHashMap<>();
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// check if any records got errors in the customizer - if so, remove them from the input list of pkeys to delete //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(customizerResult != null)
{
recordsWithValidationErrors = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get());
}
///////////////////////////////////////////////////////////////////////////
// after all validations, run the pre-delete customizer, if there is one //
///////////////////////////////////////////////////////////////////////////
Optional<AbstractPreDeleteCustomizer> preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole());
if(preDeleteCustomizer.isPresent() && oldRecordList.isPresent())
{
////////////////////////////////////////////////////////////////////////////
// make list of records that are still good - to pass into the customizer //
////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyField, oldRecordList.get(), recordsWithValidationErrors);
preDeleteCustomizer.get().setDeleteInput(deleteInput);
List<QRecord> customizerResult = preDeleteCustomizer.get().apply(recordsForCustomizer);
///////////////////////////////////////////////////////
// check if any records got errors in the customizer //
///////////////////////////////////////////////////////
Set<Serializable> primaryKeysToRemoveFromInput = new HashSet<>();
for(QRecord record : customizerResult)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
recordsWithValidationErrors.add(record);
primaryKeysToRemoveFromInput.add(record.getValue(primaryKeyField));
primaryKeysToRemoveFromInput.add(record.getValue(primaryKeyFieldName));
}
else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
{
recordsWithValidationWarnings.add(record);
recordsWithValidationWarnings.put(record.getValue(primaryKeyFieldName), record);
}
}
/////////////////////////////////////////////////////////////////
// do one mass removal of any bad keys from the input key list //
/////////////////////////////////////////////////////////////////
if(!primaryKeysToRemoveFromInput.isEmpty())
{
deleteInput.getPrimaryKeys().removeAll(primaryKeysToRemoveFromInput);
primaryKeys.removeAll(primaryKeysToRemoveFromInput);
}
}
@ -171,24 +173,34 @@ public class DeleteAction
////////////////////////////////////
DeleteOutput deleteOutput = deleteInterface.execute(deleteInput);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// merge the backend's output with any validation errors we found (whose ids wouldn't have gotten into the backend delete) //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
if(outputRecordsWithErrors == null)
{
deleteOutput.setRecordsWithErrors(new ArrayList<>());
outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// merge the backend's output with any validation errors we found (whose pkeys wouldn't have gotten into the backend delete) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> outputRecordsWithErrors = Objects.requireNonNullElseGet(deleteOutput.getRecordsWithErrors(), () -> new ArrayList<>());
outputRecordsWithErrors.addAll(recordsWithValidationErrors);
List<QRecord> outputRecordsWithWarnings = deleteOutput.getRecordsWithWarnings();
if(outputRecordsWithWarnings == null)
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if a record had a validation warning, but then an execution error, remove it from the warning list - so it's only in one of them. //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord outputRecordWithError : outputRecordsWithErrors)
{
deleteOutput.setRecordsWithWarnings(new ArrayList<>());
outputRecordsWithWarnings = deleteOutput.getRecordsWithWarnings();
Serializable pkey = outputRecordWithError.getValue(primaryKeyFieldName);
recordsWithValidationWarnings.remove(pkey);
}
///////////////////////////////////////////////////////////////////////////////////////////
// combine the warning list from validation to that from execution - avoiding duplicates //
// use a map to manage this list for the rest of this method //
///////////////////////////////////////////////////////////////////////////////////////////
Map<Serializable, QRecord> outputRecordsWithWarningMap = CollectionUtils.nullSafeIsEmpty(deleteOutput.getRecordsWithWarnings()) ? new LinkedHashMap<>()
: deleteOutput.getRecordsWithWarnings().stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyFieldName), r -> r, (a, b) -> a, () -> new LinkedHashMap<>()));
for(Map.Entry<Serializable, QRecord> entry : recordsWithValidationWarnings.entrySet())
{
if(!outputRecordsWithWarningMap.containsKey(entry.getKey()))
{
outputRecordsWithWarningMap.put(entry.getKey(), entry.getValue());
}
}
outputRecordsWithWarnings.addAll(recordsWithValidationWarnings);
////////////////////////////////////////
// delete associations, if applicable //
@ -212,25 +224,27 @@ public class DeleteAction
////////////////////////////////////////////////////////////////////////////
// make list of records that are still good - to pass into the customizer //
////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyField, oldRecordList.get(), outputRecordsWithErrors);
List<QRecord> recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyFieldName, oldRecordList.get(), outputRecordsWithErrors);
try
{
postDeleteCustomizer.get().setDeleteInput(deleteInput);
List<QRecord> customizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer);
List<QRecord> postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer);
///////////////////////////////////////////////////////
// check if any records got errors in the customizer //
///////////////////////////////////////////////////////
for(QRecord record : customizerResult)
for(QRecord record : postCustomizerResult)
{
Serializable pkey = record.getValue(primaryKeyFieldName);
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
outputRecordsWithErrors.add(record);
outputRecordsWithWarningMap.remove(pkey);
}
else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
{
outputRecordsWithWarnings.add(record);
outputRecordsWithWarningMap.put(pkey, record);
}
}
}
@ -239,16 +253,65 @@ public class DeleteAction
for(QRecord record : recordsForCustomizer)
{
record.addWarning(new QWarningMessage("An error occurred after the delete: " + e.getMessage()));
outputRecordsWithWarnings.add(record);
outputRecordsWithWarningMap.put(record.getValue(primaryKeyFieldName), record);
}
}
}
deleteOutput.setRecordsWithErrors(outputRecordsWithErrors);
deleteOutput.setRecordsWithWarnings(new ArrayList<>(outputRecordsWithWarningMap.values()));
return deleteOutput;
}
/*******************************************************************************
** this method takes in the deleteInput, and the list of old records that matched
** the pkeys in that input.
**
** it'll check if any of those pkeys aren't found (in a sub-method) - a record
** with an error message will be added to oldRecordList for any such records.
**
** it'll also then call the pre-customizer, if there is one - taking in the
** oldRecordList. it can add other errors or warnings to records.
**
** The return value here is basically oldRecordList - possibly with some new
** entries for the pkey-not-founds, and possibly w/ errors and warnings from the
** customizer.
*******************************************************************************/
public List<QRecord> performValidations(DeleteInput deleteInput, Optional<List<QRecord>> oldRecordList, boolean isPreview) throws QException
{
if(oldRecordList.isEmpty())
{
return (null);
}
QTableMetaData table = deleteInput.getTable();
List<QRecord> primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get());
///////////////////////////////////////////////////////////////////////////
// after all validations, run the pre-delete customizer, if there is one //
///////////////////////////////////////////////////////////////////////////
Optional<AbstractPreDeleteCustomizer> preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole());
List<QRecord> customizerResult = oldRecordList.get();
if(preDeleteCustomizer.isPresent())
{
preDeleteCustomizer.get().setDeleteInput(deleteInput);
preDeleteCustomizer.get().setIsPreview(isPreview);
customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get());
}
/////////////////////////////////////////////////////////////////////////
// add any pkey-not-found records to the front of the customizerResult //
/////////////////////////////////////////////////////////////////////////
customizerResult.addAll(primaryKeysNotFound);
return customizerResult;
}
/*******************************************************************************
**
*******************************************************************************/
@ -344,9 +407,9 @@ public class DeleteAction
** records that you can't see because of security - that they won't be found
** by the query here, so it's the same to you as if they don't exist at all!
**
** This method, if it finds any missing records, will:
** - remove those ids from the deleteInput
** - create a QRecord with that id and a not-found error message.
** If this method identifies any missing records (e.g., from PKeys that are
** requested to be deleted, but don't exist (or can't be seen)), then it will
** return those as new QRecords, with error messages.
*******************************************************************************/
private List<QRecord> validateRecordsExistAndCanBeAccessed(DeleteInput deleteInput, List<QRecord> oldRecordList) throws QException
{
@ -355,64 +418,28 @@ public class DeleteAction
QTableMetaData table = deleteInput.getTable();
QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
Set<Serializable> primaryKeysToRemoveFromInput = new HashSet<>();
List<List<Serializable>> pages = CollectionUtils.getPages(deleteInput.getPrimaryKeys(), 1000);
for(List<Serializable> page : pages)
{
List<Serializable> primaryKeysToLookup = new ArrayList<>();
for(Serializable primaryKeyValue : page)
{
if(primaryKeyValue != null)
{
primaryKeysToLookup.add(primaryKeyValue);
}
}
Map<Serializable, QRecord> lookedUpRecords = new HashMap<>();
if(CollectionUtils.nullSafeHasContents(oldRecordList))
{
Map<Serializable, QRecord> oldRecordMapByPrimaryKey = new HashMap<>();
for(QRecord record : oldRecordList)
{
Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField());
primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
lookedUpRecords.put(primaryKeyValue, record);
}
}
else if(!primaryKeysToLookup.isEmpty())
{
QueryInput queryInput = new QueryInput();
queryInput.setTransaction(deleteInput.getTransaction());
queryInput.setTableName(table.getName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeysToLookup)));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
for(QRecord record : queryOutput.getRecords())
{
lookedUpRecords.put(record.getValue(table.getPrimaryKeyField()), record);
}
oldRecordMapByPrimaryKey.put(primaryKeyValue, record);
}
for(Serializable primaryKeyValue : page)
{
primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
if(!lookedUpRecords.containsKey(primaryKeyValue))
if(!oldRecordMapByPrimaryKey.containsKey(primaryKeyValue))
{
QRecord recordWithError = new QRecord();
recordsWithErrors.add(recordWithError);
recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue);
recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue));
primaryKeysToRemoveFromInput.add(primaryKeyValue);
}
}
/////////////////////////////////////////////////////////////////
// do one mass removal of any bad keys from the input key list //
/////////////////////////////////////////////////////////////////
if(!primaryKeysToRemoveFromInput.isEmpty())
{
deleteInput.getPrimaryKeys().removeAll(primaryKeysToRemoveFromInput);
primaryKeysToRemoveFromInput.clear();
}
}
return (recordsWithErrors);

41
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -101,20 +101,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
/////////////////////////////
// run standard validators //
/////////////////////////////
ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
setErrorsIfUniqueKeyErrors(insertInput, table);
validateRequiredFields(insertInput);
ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
///////////////////////////////////////////////////////////////////////////
// after all validations, run the pre-insert customizer, if there is one //
///////////////////////////////////////////////////////////////////////////
Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
if(preInsertCustomizer.isPresent())
{
preInsertCustomizer.get().setInsertInput(insertInput);
insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
}
performValidations(insertInput, false);
////////////////////////////////////
// have the backend do the insert //
@ -172,6 +159,32 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
/*******************************************************************************
**
*******************************************************************************/
public void performValidations(InsertInput insertInput, boolean isPreview) throws QException
{
QTableMetaData table = insertInput.getTable();
ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
setErrorsIfUniqueKeyErrors(insertInput, table);
validateRequiredFields(insertInput);
ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
///////////////////////////////////////////////////////////////////////////
// after all validations, run the pre-insert customizer, if there is one //
///////////////////////////////////////////////////////////////////////////
Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
if(preInsertCustomizer.isPresent())
{
preInsertCustomizer.get().setInsertInput(insertInput);
preInsertCustomizer.get().setIsPreview(isPreview);
insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
}
}
/*******************************************************************************
**
*******************************************************************************/

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -65,13 +65,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QMiddlewareTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -808,7 +808,7 @@ public class QInstanceEnricher
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
ExtractViaQueryStep.class,
BulkDeleteTransformStep.class,
LoadViaDeleteStep.class,
BulkDeleteLoadStep.class,
values
)
.withName(processName)

156
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java Normal file
View File

@ -0,0 +1,156 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
** Generic implementation of a LoadStep - that runs a Delete action for a
** specified table.
*******************************************************************************/
public class BulkDeleteLoadStep extends LoadViaDeleteStep implements ProcessSummaryProviderInterface
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private List<ProcessSummaryLine> infoSummaries = new ArrayList<>();
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("deleted");
private String tableLabel;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
super.preRun(runBackendStepInput, runBackendStepOutput);
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
if(table != null)
{
tableLabel = table.getLabel();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
{
ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings";
okSummary.setSingularPastMessage(tableLabel + " record was deleted" + noWarningsSuffix + ".");
okSummary.setPluralPastMessage(tableLabel + " records were deleted" + noWarningsSuffix + ".");
okSummary.pickMessage(isForResultScreen);
okSummary.addSelfToListIfAnyCount(rs);
processSummaryWarningsAndErrorsRollup.addToList(rs);
rs.addAll(infoSummaries);
return (rs);
}
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////////
// have base class delete //
////////////////////////////
super.run(runBackendStepInput, runBackendStepOutput);
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
String primaryKeyFieldName = table.getPrimaryKeyField();
Map<Serializable, QRecord> outputRecordMap = runBackendStepOutput.getRecords().stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyFieldName), r -> r, (a, b) -> a));
///////////////////////////////////////////////////////////////////////////////////////////////////
// roll up the results, based on the input list, but looking for error/warnings from output list //
///////////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
{
Serializable recordPrimaryKey = record.getValue(primaryKeyFieldName);
QRecord outputRecord = outputRecordMap.get(recordPrimaryKey);
if(CollectionUtils.nullSafeHasContents(outputRecord.getErrors()))
{
String message = outputRecord.getErrors().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addError(message, recordPrimaryKey);
}
else if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings()))
{
String message = outputRecord.getWarnings().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addWarning(message, recordPrimaryKey);
}
else
{
okSummary.incrementCountAndAddPrimaryKey(recordPrimaryKey);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(new InsertAction().openTransaction(insertInput)));
}
}

83
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java
View File

@ -22,16 +22,24 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
@ -41,6 +49,8 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("deleted");
private String tableLabel;
@ -69,11 +79,15 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
QTableMetaData table = runBackendStepInput.getTable();
String primaryKeyField = table.getPrimaryKeyField();
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// on the validate step, we haven't read the full file, so we don't know how many rows there are - thus //
// record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) ||
runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_PREVIEW))
{
if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null)
{
@ -83,6 +97,42 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record");
}
///////////////////////////////////////////////////////////////////////
// run the validation - critically - in preview mode (boolean param) //
///////////////////////////////////////////////////////////////////////
DeleteAction deleteAction = new DeleteAction();
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(runBackendStepInput.getTableName());
deleteInput.setPrimaryKeys(runBackendStepInput.getRecords().stream().map(r -> r.getValue(primaryKeyField)).toList());
List<QRecord> validationResultRecords = deleteAction.performValidations(deleteInput, Optional.of(runBackendStepInput.getRecords()), true);
/////////////////////////////////////////////////////////////
// look at the update input to build process summary lines //
/////////////////////////////////////////////////////////////
List<QRecord> outputRecords = new ArrayList<>();
for(QRecord record : validationResultRecords)
{
Serializable recordPrimaryKey = record.getValue(table.getPrimaryKeyField());
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
String message = record.getErrors().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addError(message, recordPrimaryKey);
}
else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
{
String message = record.getWarnings().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addWarning(message, recordPrimaryKey);
outputRecords.add(record);
}
else
{
okSummary.incrementCountAndAddPrimaryKey(recordPrimaryKey);
outputRecords.add(record);
}
}
runBackendStepOutput.setRecords(outputRecords);
}
else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE))
{
@ -94,13 +144,15 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " records");
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// no transformation needs done - just pass records through from input to output, and assume all are OK //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// no transformation needs done - just pass records through from input to output, and assume errors & warnings will come from the delete action //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
okSummary.incrementCount(runBackendStepInput.getRecords().size());
// i think load step will do this.
// okSummary.incrementCount(runBackendStepInput.getRecords().size());
}
}
@ -111,17 +163,16 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
{
if(isForResultScreen)
{
okSummary.setMessage(tableLabel + " records were deleted.");
}
else
{
okSummary.setMessage(tableLabel + " records will be deleted.");
}
ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
rs.add(okSummary);
String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings";
okSummary.setSingularFutureMessage(tableLabel + " record will be deleted" + noWarningsSuffix + ".");
okSummary.setPluralFutureMessage(tableLabel + " records will be deleted" + noWarningsSuffix + ".");
okSummary.pickMessage(isForResultScreen);
okSummary.addSelfToListIfAnyCount(rs);
processSummaryWarningsAndErrorsRollup.addToList(rs);
return (rs);
}
}

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java
View File

@ -38,7 +38,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.buildInfoSummaryLines;
import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.getProcessSummaryWarningsAndErrorsRollup;
/*******************************************************************************
@ -51,7 +50,7 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private List<ProcessSummaryLine> infoSummaries = new ArrayList<>();
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = getProcessSummaryWarningsAndErrorsRollup();
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited");
private String tableLabel;
@ -104,9 +103,15 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
////////////////////////////
// have base class update //
////////////////////////////
super.run(runBackendStepInput, runBackendStepOutput);
////////////////////////////////////////////////////////
// roll up results based on output from update action //
////////////////////////////////////////////////////////
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
for(QRecord record : runBackendStepOutput.getRecords())
{
Serializable recordPrimaryKey = record.getValue(table.getPrimaryKeyField());

35
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java
View File

@ -59,7 +59,7 @@ public class BulkEditTransformStep extends AbstractTransformStep
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private List<ProcessSummaryLine> infoSummaries = new ArrayList<>();
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = getProcessSummaryWarningsAndErrorsRollup();
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited");
private QTableMetaData table;
private String tableLabel;
@ -71,36 +71,6 @@ public class BulkEditTransformStep extends AbstractTransformStep
/*******************************************************************************
** used by Load step too
*******************************************************************************/
static ProcessSummaryWarningsAndErrorsRollup getProcessSummaryWarningsAndErrorsRollup()
{
return new ProcessSummaryWarningsAndErrorsRollup()
.withErrorTemplate(new ProcessSummaryLine(Status.ERROR)
.withSingularFutureMessage("record has an error: ")
.withPluralFutureMessage("records have an error: ")
.withSingularPastMessage("record had an error: ")
.withPluralPastMessage("records had an error: "))
.withWarningTemplate(new ProcessSummaryLine(Status.WARNING)
.withSingularFutureMessage("record will be edited, but has a warning: ")
.withPluralFutureMessage("records will be edited, but have a warning: ")
.withSingularPastMessage("record was edited, but had a warning: ")
.withPluralPastMessage("records were edited, but had a warning: "))
.withOtherErrorsSummary(new ProcessSummaryLine(Status.ERROR)
.withSingularFutureMessage("record has an other error.")
.withPluralFutureMessage("records have other errors.")
.withSingularPastMessage("record had an other error.")
.withPluralPastMessage("records had other errors."))
.withOtherWarningsSummary(new ProcessSummaryLine(Status.WARNING)
.withSingularFutureMessage("record will be edited, but has an other warning.")
.withPluralFutureMessage("records will be edited, but have other warnings.")
.withSingularPastMessage("record was edited, but had other warnings.")
.withPluralPastMessage("records were edited, but had other warnings."));
}
/*******************************************************************************
**
*******************************************************************************/
@ -169,7 +139,8 @@ public class BulkEditTransformStep extends AbstractTransformStep
setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, recordToUpdate);
}
okSummary.incrementCount(runBackendStepInput.getRecords().size());
// i think load step will do this.
// okSummary.incrementCount(runBackendStepInput.getRecords().size());
}
else
{

84
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -30,6 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -52,6 +55,9 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
public class BulkInsertTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted");
private Map<UniqueKey, ProcessSummaryLine> ukErrorSummaries = new HashMap<>();
private QTableMetaData table;
@ -112,20 +118,24 @@ public class BulkInsertTransformStep extends AbstractTransformStep
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// no transformation needs to be done - just pass records through from input to output, if they don't violate any UK's //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction //
// will only be getting the records in pages, but in here, we'll track UK's across pages!! //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////
// if there are no UK's, just output all records //
///////////////////////////////////////////////////
////////////////////////////////////////////////////
// if there are no UK's, proceed with all records //
////////////////////////////////////////////////////
List<QRecord> recordsWithoutUkErrors = new ArrayList<>();
if(existingKeys.isEmpty())
{
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
okSummary.incrementCount(runBackendStepInput.getRecords().size());
recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords());
}
else
{
/////////////////////////////////////////////////////////////
// else, only proceed with records that don't violate a UK //
/////////////////////////////////////////////////////////////
for(UniqueKey uniqueKey : uniqueKeys)
{
keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>());
@ -162,11 +172,47 @@ public class BulkInsertTransformStep extends AbstractTransformStep
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv));
}
okSummary.incrementCount();
runBackendStepOutput.addRecord(record);
recordsWithoutUkErrors.add(record);
}
}
}
/////////////////////////////////////////////////////////////////////////////////
// run all validation from the insert action - in Preview mode (boolean param) //
/////////////////////////////////////////////////////////////////////////////////
InsertAction insertAction = new InsertAction();
InsertInput insertInput = new InsertInput();
insertInput.setTableName(runBackendStepInput.getTableName());
insertInput.setRecords(recordsWithoutUkErrors);
insertInput.setSkipUniqueKeyCheck(true);
insertAction.performValidations(insertInput, true);
List<QRecord> validationResultRecords = insertInput.getRecords();
/////////////////////////////////////////////////////////////////
// look at validation results to build process summary results //
/////////////////////////////////////////////////////////////////
List<QRecord> outputRecords = new ArrayList<>();
for(QRecord record : validationResultRecords)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
String message = record.getErrors().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addError(message, null);
}
else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
{
String message = record.getWarnings().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addWarning(message, null);
outputRecords.add(record);
}
else
{
okSummary.incrementCountAndAddPrimaryKey(null);
outputRecords.add(record);
}
}
runBackendStepOutput.setRecords(outputRecords);
}
@ -177,22 +223,22 @@ public class BulkInsertTransformStep extends AbstractTransformStep
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
{
ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
String tableLabel = table == null ? "" : table.getLabel();
okSummary
.withSingularFutureMessage(tableLabel + " record will be inserted")
.withPluralFutureMessage(tableLabel + " records will be inserted")
.withSingularPastMessage(tableLabel + " record was inserted")
.withPluralPastMessage(tableLabel + " records were inserted");
ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings";
okSummary.setSingularFutureMessage(tableLabel + " record will be inserted" + noWarningsSuffix + ".");
okSummary.setPluralFutureMessage(tableLabel + " records will be inserted" + noWarningsSuffix + ".");
okSummary.setSingularPastMessage(tableLabel + " record was inserted" + noWarningsSuffix + ".");
okSummary.setPluralPastMessage(tableLabel + " records were inserted" + noWarningsSuffix + ".");
okSummary.pickMessage(isForResultScreen);
okSummary.addSelfToListIfAnyCount(rs);
for(Map.Entry<UniqueKey, ProcessSummaryLine> entry : ukErrorSummaries.entrySet())
{
UniqueKey uniqueKey = entry.getKey();
ProcessSummaryLine ukErrorSummary = entry.getValue();
String ukErrorSuffix = " inserted, because they contain a duplicate key (" + uniqueKey.getDescription(table) + ")";
String ukErrorSuffix = " inserted, because of duplicate values in a unique key (" + uniqueKey.getDescription(table) + ")";
ukErrorSummary
.withSingularFutureMessage(tableLabel + " record will not be" + ukErrorSuffix)
@ -203,6 +249,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
ukErrorSummary.addSelfToListIfAnyCount(rs);
}
processSummaryWarningsAndErrorsRollup.addToList(rs);
return (rs);
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java
View File

@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -60,7 +61,9 @@ public class LoadViaDeleteStep extends AbstractLoadStep
deleteInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
// todo? can make more efficient deletes, maybe? deleteInput.setQueryFilter();
getTransaction().ifPresent(deleteInput::setTransaction);
new DeleteAction().execute(deleteInput);
DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
runBackendStepOutput.getRecords().addAll(deleteOutput.getRecordsWithErrors());
runBackendStepOutput.getRecords().addAll(deleteOutput.getRecordsWithWarnings());
}

38
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java
View File

@ -48,6 +48,36 @@ public class ProcessSummaryWarningsAndErrorsRollup
/*******************************************************************************
**
*******************************************************************************/
public static ProcessSummaryWarningsAndErrorsRollup build(String pastTenseVerb)
{
return new ProcessSummaryWarningsAndErrorsRollup()
.withErrorTemplate(new ProcessSummaryLine(Status.ERROR)
.withSingularFutureMessage("record has an error: ")
.withPluralFutureMessage("records have an error: ")
.withSingularPastMessage("record had an error: ")
.withPluralPastMessage("records had an error: "))
.withWarningTemplate(new ProcessSummaryLine(Status.WARNING)
.withSingularFutureMessage("record will be " + pastTenseVerb + ", but has a warning: ")
.withPluralFutureMessage("records will be " + pastTenseVerb + ", but have a warning: ")
.withSingularPastMessage("record was " + pastTenseVerb + ", but had a warning: ")
.withPluralPastMessage("records were " + pastTenseVerb + ", but had a warning: "))
.withOtherErrorsSummary(new ProcessSummaryLine(Status.ERROR)
.withSingularFutureMessage("record has an other error.")
.withPluralFutureMessage("records have other errors.")
.withSingularPastMessage("record had an other error.")
.withPluralPastMessage("records had other errors."))
.withOtherWarningsSummary(new ProcessSummaryLine(Status.WARNING)
.withSingularFutureMessage("record will be " + pastTenseVerb + ", but has an other warning.")
.withPluralFutureMessage("records will be " + pastTenseVerb + ", but have other warnings.")
.withSingularPastMessage("record was " + pastTenseVerb + ", but had other warnings.")
.withPluralPastMessage("records were " + pastTenseVerb + ", but had other warnings."));
}
/*******************************************************************************
**
*******************************************************************************/
@ -164,8 +194,16 @@ public class ProcessSummaryWarningsAndErrorsRollup
}
}
}
if(primaryKey == null)
{
processSummaryLine.incrementCount();
}
else
{
processSummaryLine.incrementCountAndAddPrimaryKey(primaryKey);
}
}