diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java index 5c9f0e14..9cdefa74 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java @@ -57,7 +57,7 @@ public class BulkTableActionProcessPermissionChecker implements CustomPermission switch(bulkActionName) { case "bulkInsert" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.INSERT); - case "bulkEdit" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.EDIT); + case "bulkEdit", "bulkEditWithFile" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.EDIT); case "bulkDelete" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.DELETE); default -> LOG.warn("Unexpected bulk action name when checking permissions for process: " + processName); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 946622f4..5b2486f8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.bulk.TableKeyFieldsPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData; @@ -901,6 +902,11 @@ public class QInstanceEnricher *******************************************************************************/ private void defineTableBulkProcesses(QInstance qInstance) { + if(qInstance.getPossibleValueSource(TableKeyFieldsPossibleValueSource.NAME) == null) + { + qInstance.addPossibleValueSource(defineTableKeyFieldsPossibleValueSource()); + } + for(QTableMetaData table : qInstance.getTables().values()) { if(table.getFields() == null) @@ -924,6 +930,12 @@ public class QInstanceEnricher defineTableBulkEdit(qInstance, table, bulkEditProcessName); } + String bulkEditWithFileProcessName = table.getName() + ".bulkEditWithFile"; + if(qInstance.getProcess(bulkEditWithFileProcessName) == null) + { + defineTableBulkEditWithFile(qInstance, table, bulkEditWithFileProcessName); + } + String bulkDeleteProcessName = table.getName() + ".bulkDelete"; if(qInstance.getProcess(bulkDeleteProcessName) == null) { @@ -1104,6 +1116,122 @@ public class QInstanceEnricher + /******************************************************************************* + ** + *******************************************************************************/ + public void defineTableBulkEditWithFile(QInstance qInstance, QTableMetaData table, String processName) + { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "This is a preview of the records that will be updated."); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + BulkInsertExtractStep.class, + BulkInsertTransformStep.class, + BulkEditLoadStep.class, + values + ) + .withName(processName) + .withLabel(table.getLabel() + " Bulk Edit With File") + .withTableName(table.getName()) + .withIsHidden(true) + .withPermissionRules(qInstance.getDefaultPermissionRules().clone() + .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class))); + + List editableFields = table.getFields().values().stream() + .filter(QFieldMetaData::getIsEditable) + .filter(f -> !f.getType().equals(QFieldType.BLOB)) + .toList(); + + QBackendStepMetaData prepareFileUploadStep = new QBackendStepMetaData() + .withName("prepareFileUpload") + .withCode(new QCodeReference(BulkInsertPrepareFileUploadStep.class)); + + QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() + .withName("upload") + .withLabel("Upload File") + .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB) + .withFieldAdornment(FileUploadAdornment.newFieldAdornment() + .withValue(FileUploadAdornment.formatDragAndDrop()) + .withValue(FileUploadAdornment.widthFull())) + .withLabel(table.getLabel() + " File") + .withIsRequired(true)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); + + QBackendStepMetaData prepareFileMappingStep = new QBackendStepMetaData() + .withName("prepareFileMapping") + .withCode(new QCodeReference(BulkInsertPrepareFileMappingStep.class)); + + QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() + .withName("fileMapping") + .withLabel("File Mapping") + .withBackStepName("prepareFileUpload") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)) + .withFormField(new QFieldMetaData("hasHeaderRow", QFieldType.BOOLEAN)) + .withFormField(new QFieldMetaData("layout", QFieldType.STRING)) // is actually PVS, but, this field is only added to help support helpContent, so :shrug: + .withFormField(new QFieldMetaData("tableKeyFields", QFieldType.STRING).withPossibleValueSourceName(TableKeyFieldsPossibleValueSource.NAME)); + + QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData() + .withName("receiveFileMapping") + .withCode(new QCodeReference(BulkInsertReceiveFileMappingStep.class)); + + QBackendStepMetaData prepareValueMappingStep = new QBackendStepMetaData() + .withName("prepareValueMapping") + .withCode(new QCodeReference(BulkInsertPrepareValueMappingStep.class)); + + QFrontendStepMetaData valueMappingScreen = new QFrontendStepMetaData() + .withName("valueMapping") + .withLabel("Value Mapping") + .withBackStepName("prepareFileMapping") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM)); + + QBackendStepMetaData receiveValueMappingStep = new QBackendStepMetaData() + .withName("receiveValueMapping") + .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class)); + + int i = 0; + process.withStep(i++, prepareFileUploadStep); + process.withStep(i++, uploadScreen); + + process.withStep(i++, prepareFileMappingStep); + process.withStep(i++, fileMappingScreen); + process.withStep(i++, receiveFileMappingStep); + + process.withStep(i++, prepareValueMappingStep); + process.withStep(i++, valueMappingScreen); + process.withStep(i++, receiveValueMappingStep); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields); + + ////////////////////////////////////////////////////////////////////////////////////////// + // put the bulk-load profile form (e.g., for saving it) on the review & result screens) // + ////////////////////////////////////////////////////////////////////////////////////////// + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW) + .withBackStepName("prepareFileMapping") + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_RESULT) + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + + qInstance.addProcess(process); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QPossibleValueSource defineTableKeyFieldsPossibleValueSource() + { + return (new QPossibleValueSource() + .withName(TableKeyFieldsPossibleValueSource.NAME) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(TableKeyFieldsPossibleValueSource.class))); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java index 6bbb3dc9..df934c4a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java @@ -35,11 +35,13 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; *******************************************************************************/ public class SearchPossibleValueSourceInput extends AbstractActionInput implements Cloneable { - private String possibleValueSourceName; - private QQueryFilter defaultQueryFilter; - private String searchTerm; - private List idList; - private List labelList; + private String possibleValueSourceName; + private QQueryFilter defaultQueryFilter; + private String searchTerm; + private List idList; + private List labelList; + private Map pathParamMap; + private Map> queryParamMap; private Map otherValues; @@ -319,6 +321,68 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen + /******************************************************************************* + ** Getter for pathParamMap + *******************************************************************************/ + public Map getPathParamMap() + { + return (this.pathParamMap); + } + + + + /******************************************************************************* + ** Setter for pathParamMap + *******************************************************************************/ + public void setPathParamMap(Map pathParamMap) + { + this.pathParamMap = pathParamMap; + } + + + + /******************************************************************************* + ** Fluent setter for pathParamMap + *******************************************************************************/ + public SearchPossibleValueSourceInput withPathParamMap(Map pathParamMap) + { + this.pathParamMap = pathParamMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for queryParamMap + *******************************************************************************/ + public Map> getQueryParamMap() + { + return (this.queryParamMap); + } + + + + /******************************************************************************* + ** Setter for queryParamMap + *******************************************************************************/ + public void setQueryParamMap(Map> queryParamMap) + { + this.queryParamMap = queryParamMap; + } + + + + /******************************************************************************* + ** Fluent setter for queryParamMap + *******************************************************************************/ + public SearchPossibleValueSourceInput withQueryParamMap(Map> queryParamMap) + { + this.queryParamMap = queryParamMap; + return (this); + } + + + /******************************************************************************* ** Getter for otherValues *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/bulk/TableKeyFieldsPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/bulk/TableKeyFieldsPossibleValueSource.java new file mode 100644 index 00000000..45e42a84 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/bulk/TableKeyFieldsPossibleValueSource.java @@ -0,0 +1,153 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.bulk; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TableKeyFieldsPossibleValueSource implements QCustomPossibleValueProvider +{ + public static final String NAME = "tableKeyFields"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable tableAndKey) + { + QPossibleValue possibleValue = null; + + ///////////////////////////////////////////////////////////// + // keys are in the format -|| // + ///////////////////////////////////////////////////////////// + String[] keyParts = tableAndKey.toString().split("-"); + String tableName = keyParts[0]; + String key = keyParts[1]; + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table.getPrimaryKeyField().equals(key)) + { + String id = table.getPrimaryKeyField(); + String label = table.getField(table.getPrimaryKeyField()).getLabel(); + possibleValue = new QPossibleValue<>(id, label); + } + else + { + for(UniqueKey uniqueKey : table.getUniqueKeys()) + { + String potentialMatch = getIdFromUniqueKey(uniqueKey); + if(potentialMatch.equals(key)) + { + String id = potentialMatch; + String label = getLabelFromUniqueKey(table, uniqueKey); + possibleValue = new QPossibleValue<>(id, label); + break; + } + } + } + + return (possibleValue); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + List> rs = new ArrayList<>(); + if(!CollectionUtils.nonNullMap(input.getPathParamMap()).containsKey("processName") || input.getPathParamMap().get("processName") == null || input.getPathParamMap().get("processName").isEmpty()) + { + throw (new QException("Path Param of processName was not found.")); + } + + //////////////////////////////////////////////////// + // process name will be like tnt.bulkEditWithFile // + //////////////////////////////////////////////////// + String processName = input.getPathParamMap().get("processName"); + String tableName = processName.split("\\.")[0]; + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys())) + { + String id = getIdFromUniqueKey(uniqueKey); + String label = getLabelFromUniqueKey(table, uniqueKey); + if(!StringUtils.hasContent(input.getSearchTerm()) || input.getSearchTerm().equals(id)) + { + rs.add(new QPossibleValue<>(id, label)); + } + } + rs.sort(Comparator.comparing(QPossibleValue::getLabel)); + + /////////////////////////////// + // put the primary key first // + /////////////////////////////// + if(!StringUtils.hasContent(input.getSearchTerm()) || input.getSearchTerm().equals(table.getPrimaryKeyField())) + { + rs.add(0, new QPossibleValue<>(table.getPrimaryKeyField(), table.getField(table.getPrimaryKeyField()).getLabel())); + } + + return rs; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getIdFromUniqueKey(UniqueKey uniqueKey) + { + return (StringUtils.join("|", uniqueKey.getFieldNames())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getLabelFromUniqueKey(QTableMetaData tableMetaData, UniqueKey uniqueKey) + { + List fieldLabels = new ArrayList<>(uniqueKey.getFieldNames().stream().map(f -> tableMetaData.getField(f).getLabel()).toList()); + fieldLabels.sort(Comparator.naturalOrder()); + return (StringUtils.joinWithCommasAndAnd(fieldLabels)); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java index 65f07c77..092dfd85 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java @@ -60,6 +60,9 @@ public class SavedBulkLoadProfile extends QRecordEntity @QField(label = "Mapping JSON") private String mappingJson; + @QField() + private Boolean isBulkEdit; + /******************************************************************************* @@ -251,7 +254,6 @@ public class SavedBulkLoadProfile extends QRecordEntity - /******************************************************************************* ** Getter for mappingJson *******************************************************************************/ @@ -282,4 +284,34 @@ public class SavedBulkLoadProfile extends QRecordEntity } + + /******************************************************************************* + ** Getter for isBulkEdit + *******************************************************************************/ + public Boolean getIsBulkEdit() + { + return (this.isBulkEdit); + } + + + + /******************************************************************************* + ** Setter for isBulkEdit + *******************************************************************************/ + public void setIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + } + + + + /******************************************************************************* + ** Fluent setter for isBulkEdit + *******************************************************************************/ + public SavedBulkLoadProfile withIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java index 1438dd37..ddc3e062 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java @@ -113,7 +113,7 @@ public class SavedBulkLoadProfileMetaDataProvider .withFieldsFromEntity(SavedBulkLoadProfile.class) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) - .withSection(new QFieldSection("details", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "mappingJson"))) + .withSection(new QFieldSection("details", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "mappingJson", "isBulkEdit"))) .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); table.getField("mappingJson").withBehavior(SavedBulkLoadProfileJsonFieldDisplayValueFormatter.getInstance()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java index 72eaa3fb..abf9097c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java @@ -36,10 +36,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.commons.lang.BooleanUtils; import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.buildInfoSummaryLines; @@ -53,6 +55,9 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); private List infoSummaries = new ArrayList<>(); + private Serializable firstInsertedPrimaryKey = null; + private Serializable lastInsertedPrimaryKey = null; + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited"); private String tableLabel; @@ -106,7 +111,15 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar tableLabel = table.getLabel(); } - buildInfoSummaryLines(runBackendStepInput, table, infoSummaries, true); + boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit")); + if(isBulkEdit) + { + buildBulkUpdateWithFileInfoSummaryLines(runBackendStepOutput, table); + } + else + { + buildInfoSummaryLines(runBackendStepInput, table, infoSummaries, true); + } } @@ -146,4 +159,83 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar } } + + + /*************************************************************************** + ** + ***************************************************************************/ + private void buildBulkUpdateWithFileInfoSummaryLines(RunBackendStepOutput runBackendStepOutput, QTableMetaData table) + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // the transform step builds summary lines that it predicts will update successfully. // + // but those lines don't have ids, which we'd like to have (e.g., for a process trace that // + // might link to the built record). also, it's possible that there was a fail that only // + // happened in the actual update, so, basically, re-do the summary here // + ///////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep(); + ProcessSummaryLine okSummary = transformStep.okSummary; + okSummary.setCount(0); + okSummary.setPrimaryKeys(new ArrayList<>()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // but - since errors from the transform step don't even make it through to us here in the load step, // + // do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows: // + // clear out its warnings - we'll completely rebuild them here (with primary keys) // + // and add new error lines, e.g., in case of errors that only happened past the validation if possible. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup; + processSummaryWarningsAndErrorsRollup.resetWarnings(); + + List updatedRecords = runBackendStepOutput.getRecords(); + for(QRecord updatedRecord : updatedRecords) + { + Serializable primaryKey = updatedRecord.getValue(table.getPrimaryKeyField()); + if(CollectionUtils.nullSafeIsEmpty(updatedRecord.getErrors()) && primaryKey != null) + { + ///////////////////////////////////////////////////////////////////////// + // if the record had no errors, and we have a primary key for it, then // + // keep track of the range of primary keys (first and last) // + ///////////////////////////////////////////////////////////////////////// + if(firstInsertedPrimaryKey == null) + { + firstInsertedPrimaryKey = primaryKey; + } + + lastInsertedPrimaryKey = primaryKey; + + if(!CollectionUtils.nullSafeIsEmpty(updatedRecord.getWarnings())) + { + //////////////////////////////////////////////////////////////////////////// + // if there were warnings on the updated record, put it in a warning line // + //////////////////////////////////////////////////////////////////////////// + String message = updatedRecord.getWarnings().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey); + } + else + { + /////////////////////////////////////////////////////////////////////// + // if no warnings for the updated record, then put it in the OK line // + /////////////////////////////////////////////////////////////////////// + okSummary.incrementCountAndAddPrimaryKey(primaryKey); + } + } + else + { + ////////////////////////////////////////////////////////////////////// + // else if there were errors or no primary key, build an error line // + ////////////////////////////////////////////////////////////////////// + String message = "Failed to update"; + if(!CollectionUtils.nullSafeIsEmpty(updatedRecord.getErrors())) + { + ////////////////////////////////////////////////////////// + // use the error message from the record if we have one // + ////////////////////////////////////////////////////////// + message = updatedRecord.getErrors().get(0).getMessage(); + } + processSummaryWarningsAndErrorsRollup.addError(message, primaryKey); + } + } + + okSummary.pickMessage(true); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index 19e2dc79..5a3820d5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang.BooleanUtils; import org.json.JSONObject; @@ -65,9 +66,11 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep { buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); + boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit")); String tableName = runBackendStepInput.getValueString("tableName"); - BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName, isBulkEdit); runBackendStepOutput.addValue("tableStructure", tableStructure); + runBackendStepOutput.addValue("isBulkEdit", isBulkEdit); boolean needSuggestedMapping = true; if(runBackendStepOutput.getProcessState().getIsStepBack()) @@ -81,7 +84,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep { @SuppressWarnings("unchecked") List headerValues = (List) runBackendStepOutput.getValue("headerValues"); - buildSuggestedMapping(headerValues, getPrepopulatedValues(runBackendStepInput), tableStructure, runBackendStepOutput); + buildSuggestedMapping(isBulkEdit, headerValues, getPrepopulatedValues(runBackendStepInput), tableStructure, runBackendStepOutput); } } @@ -95,8 +98,8 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep String prepopulatedValuesJson = runBackendStepInput.getValueString("prepopulatedValues"); if(StringUtils.hasContent(prepopulatedValuesJson)) { - Map rs = new LinkedHashMap<>(); - JSONObject jsonObject = JsonUtils.toJSONObject(prepopulatedValuesJson); + Map rs = new LinkedHashMap<>(); + JSONObject jsonObject = JsonUtils.toJSONObject(prepopulatedValuesJson); for(String key : jsonObject.keySet()) { rs.put(key, jsonObject.optString(key, null)); @@ -112,16 +115,16 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep /*************************************************************************** ** ***************************************************************************/ - private void buildSuggestedMapping(List headerValues, Map prepopulatedValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) + private void buildSuggestedMapping(boolean isBulkEdit, List headerValues, Map prepopulatedValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) { BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); - BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); + BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues, isBulkEdit); if(CollectionUtils.nullSafeHasContents(prepopulatedValues)) { for(Map.Entry entry : prepopulatedValues.entrySet()) { - String fieldName = entry.getKey(); + String fieldName = entry.getKey(); boolean foundFieldInProfile = false; for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index 4b15b720..1149e279 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -65,10 +65,12 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep runBackendStepOutput.addValue("theFile", null); } + boolean isBulkEdit = runBackendStepInput.getProcessName().endsWith("EditWithFile"); String tableName = runBackendStepInput.getValueString("tableName"); QTableMetaData table = QContext.getQInstance().getTable(tableName); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); runBackendStepOutput.addValue("tableStructure", tableStructure); + runBackendStepOutput.addValue("isBulkEdit", isBulkEdit); List requiredFields = new ArrayList<>(); List additionalFields = new ArrayList<>(); @@ -84,6 +86,14 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep } } + ///////////////////////////////////////////// + // bulk edit allows primary key as a field // + ///////////////////////////////////////////// + if(isBulkEdit) + { + requiredFields.add(0, table.getField(table.getPrimaryKeyField())); + } + StringBuilder html; String childTableLabels = ""; @@ -96,11 +106,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep ////////////////////////////////////////////////////////////////////////////////////////////////////////////// boolean listFieldsInHelpText = false; - if(!CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) + if(isBulkEdit || !CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) { html = new StringBuilder("""

Upload either a CSV or Excel (.xlsx) file, with one row for each record you want to - insert in the ${tableLabel} table.


+ ${action} in the ${tableLabel} table.


Your file can contain any number of columns. You will be prompted to map fields from the ${tableLabel} table to columns from your file or default values for all records that @@ -204,6 +214,7 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep finishCSV(flatCSV); String htmlString = html.toString() + .replace("${action}", (isBulkEdit ? "edit" : "insert")) .replace("${tableLabel}", table.getLabel()) .replace("${childTableLabels}", childTableLabels) .replace("${flatCSV}", Base64.getEncoder().encodeToString(flatCSV.toString().getBytes(StandardCharsets.UTF_8))) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index e9c09cea..e63fb0c1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -113,6 +113,8 @@ public class BulkInsertStepUtils { String layout = runBackendStepInput.getValueString("layout"); Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow"); + String keyFields = runBackendStepInput.getValueString("keyFields"); + Boolean isBulkEdit = runBackendStepInput.getValueBoolean("isBulkEdit"); ArrayList fieldList = new ArrayList<>(); @@ -127,6 +129,7 @@ public class BulkInsertStepUtils bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); + bulkLoadProfileField.setClearIfEmpty(jsonObject.optBoolean("clearIfEmpty")); if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings")) { @@ -140,6 +143,8 @@ public class BulkInsertStepUtils } BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withIsBulkEdit(isBulkEdit) + .withKeyFields(keyFields) .withVersion(version) .withFieldList(fieldList) .withHasHeaderRow(hasHeaderRow) @@ -213,7 +218,7 @@ public class BulkInsertStepUtils { return (processTracerKeyRecordMessage); } - + return (null); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index e3979a07..e3606660 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -32,12 +33,13 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -48,6 +50,11 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; @@ -68,6 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang.BooleanUtils; +import org.json.JSONArray; +import org.json.JSONObject; /******************************************************************************* @@ -75,9 +85,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class BulkInsertTransformStep extends AbstractTransformStep { - ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + public ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); - ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") + public ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") .withDoReplaceSingletonCountLinesWithSuffixOnly(false); private ListingHash errorToExampleRowValueMap = new ListingHash<>(); @@ -190,6 +200,252 @@ public class BulkInsertTransformStep extends AbstractTransformStep QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); List records = runBackendStepInput.getRecords(); + if(BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit"))) + { + handleBulkEdit(runBackendStepInput, runBackendStepOutput, records, table); + runBackendStepOutput.addValue("isBulkEdit", true); + } + else + { + handleBulkLoad(runBackendStepInput, runBackendStepOutput, records, table); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void handleBulkEdit(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List records, QTableMetaData table) throws QException + { + /////////////////////////////////////////// + // get the key fields for this bulk edit // + /////////////////////////////////////////// + String keyFieldsString = runBackendStepInput.getValueString("keyFields"); + List keyFields = Arrays.asList(keyFieldsString.split("\\|")); + + ////////////////////////////////////////////////////////////////////////// + // if the key field is the primary key, then just look up those records // + ////////////////////////////////////////////////////////////////////////// + List nonMatchingRecords = new ArrayList<>(); + List oldRecords = new ArrayList<>(); + List recordsToUpdate = new ArrayList<>(); + if(keyFields.size() == 1 && table.getPrimaryKeyField().equals(keyFields.get(0))) + { + recordsToUpdate = records; + String primaryKeyName = table.getPrimaryKeyField(); + List primaryKeys = records.stream().map(record -> record.getValue(primaryKeyName)).toList(); + oldRecords = new QueryAction().execute(new QueryInput(table.getName()).withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeys)))).getRecords(); + + /////////////////////////////////////////// + // get a set of old records primary keys // + /////////////////////////////////////////// + Set matchedPrimaryKeys = oldRecords.stream() + .map(r -> r.getValue(table.getPrimaryKeyField())) + .collect(java.util.stream.Collectors.toSet()); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // iterate over file records and if primary keys dont match, add to the non matching records list // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : records) + { + Serializable recordKey = record.getValue(table.getPrimaryKeyField()); + if(!matchedPrimaryKeys.contains(recordKey)) + { + nonMatchingRecords.add(record); + } + } + } + else + { + Set uniqueIds = new HashSet<>(); + List potentialRecords = new ArrayList<>(); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // if not using the primary key, then we will look up all records for each part of the unique key // + // and for each found, if all unique parts match we will add to our list of database records // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(String uniqueKeyPart : keyFields) + { + List values = records.stream().map(record -> record.getValue(uniqueKeyPart)).toList(); + for(QRecord databaseRecord : new QueryAction().execute(new QueryInput(table.getName()).withFilter(new QQueryFilter(new QFilterCriteria(uniqueKeyPart, QCriteriaOperator.IN, values)))).getRecords()) + { + if(!uniqueIds.contains(databaseRecord.getValue(table.getPrimaryKeyField()))) + { + potentialRecords.add(databaseRecord); + uniqueIds.add(databaseRecord.getValue(table.getPrimaryKeyField())); + } + } + } + + /////////////////////////////////////////////////////////////////////////////// + // now iterate over all of the potential records checking each unique fields // + /////////////////////////////////////////////////////////////////////////////// + fileRecordLoop: + for(QRecord fileRecord : records) + { + for(QRecord databaseRecord : potentialRecords) + { + boolean allMatch = true; + + for(String uniqueKeyPart : keyFields) + { + if(!Objects.equals(fileRecord.getValue(uniqueKeyPart), databaseRecord.getValue(uniqueKeyPart))) + { + allMatch = false; + } + } + + ////////////////////////////////////////////////////////////////////////////////////// + // if we get here with all matching, update the record from the file's primary key, // + // add it to the list to update, and continue looping over file records // + ////////////////////////////////////////////////////////////////////////////////////// + if(allMatch) + { + oldRecords.add(databaseRecord); + fileRecord.setValue(table.getPrimaryKeyField(), databaseRecord.getValue(table.getPrimaryKeyField())); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // iterate over the fields in the bulk load profile, if the value for that field is empty and the value // + // of 'clear if empty' is set to true, then update the record to update with the old record's value // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + JSONArray array = new JSONArray(runBackendStepInput.getValueString("fieldListJSON")); + for(int i = 0; i < array.length(); i++) + { + JSONObject jsonObject = array.getJSONObject(i); + String fieldName = jsonObject.optString("fieldName"); + boolean clearIfEmpty = jsonObject.optBoolean("clearIfEmpty"); + + if(fileRecord.getValue(fieldName) == null) + { + if(clearIfEmpty) + { + fileRecord.setValue(fieldName, null); + } + else + { + fileRecord.setValue(fieldName, databaseRecord.getValue(fieldName)); + } + } + } + + recordsToUpdate.add(fileRecord); + continue fileRecordLoop; + } + } + + /////////////////////////////////////////////////////////////////////////////////////// + // if we make it here, that means the record was not found, keep for logging warning // + /////////////////////////////////////////////////////////////////////////////////////// + nonMatchingRecords.add(fileRecord); + } + } + + for(QRecord missingRecord : CollectionUtils.nonNullList(nonMatchingRecords)) + { + String message = "Did not have a matching existing record."; + processSummaryWarningsAndErrorsRollup.addError(message, null); + addToErrorToExampleRowMap(message, missingRecord); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + UpdateInput updateInput = new UpdateInput(table.getName()); + updateInput.setInputSource(QInputSource.USER); + updateInput.setRecords(recordsToUpdate); + + ////////////////////////////////////////////////////////////////////// + // load the pre-insert customizer and set it up, if there is one // + // then we'll run it based on its WhenToRun value // + // we do this, in case it needs to, for example, adjust values that // + // are part of a unique key // + ////////////////////////////////////////////////////////////////////// + boolean didAlreadyRunCustomizer = false; + Optional preUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_UPDATE_RECORD.getRole()); + if(preUpdateCustomizer.isPresent()) + { + List recordsAfterCustomizer = preUpdateCustomizer.get().preUpdate(updateInput, records, true, Optional.of(oldRecords)); + runBackendStepInput.setRecords(recordsAfterCustomizer); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // so we used to have a comment here asking "do we care if the customizer runs both now, and in the validation below?" // + // when implementing Bulk Load V2, we were seeing that some customizers were adding errors to records, both now, and // + // when they ran below. so, at that time, we added this boolean, to track and avoid the double-run... // + // we could also imagine this being a setting on the pre-insert customizer, similar to its whenToRun attribute... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + didAlreadyRunCustomizer = true; + } + + ///////////////////////////////////////////////////////////////////////////////// + // run all validation from the insert action - in Preview mode (boolean param) // + ///////////////////////////////////////////////////////////////////////////////// + updateInput.setRecords(recordsToUpdate); + UpdateAction updateAction = new UpdateAction(); + updateAction.performValidations(updateInput, Optional.of(recordsToUpdate), didAlreadyRunCustomizer); + List validationResultRecords = updateInput.getRecords(); + + ///////////////////////////////////////////////////////////////// + // look at validation results to build process summary results // + ///////////////////////////////////////////////////////////////// + List outputRecords = new ArrayList<>(); + for(QRecord record : validationResultRecords) + { + List errorsFromAssociations = getErrorsFromAssociations(record); + if(CollectionUtils.nullSafeHasContents(errorsFromAssociations)) + { + List recordErrors = Objects.requireNonNullElseGet(record.getErrors(), () -> new ArrayList<>()); + recordErrors.addAll(errorsFromAssociations); + record.setErrors(recordErrors); + } + + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + for(QErrorMessage error : record.getErrors()) + { + if(error instanceof AbstractBulkLoadRollableValueError rollableValueError) + { + processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(rollableValueError, record); + } + else + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + addToErrorToExampleRowMap(error.getMessage(), record); + } + } + } + else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) + { + String message = record.getWarnings().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addWarning(message, null); + outputRecords.add(record); + } + else + { + okSummary.incrementCountAndAddPrimaryKey(null); + outputRecords.add(record); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + String associationName = entry.getKey(); + ProcessSummaryLine associationToInsertLine = associationsToInsertSummaries.computeIfAbsent(associationName, x -> new ProcessSummaryLine(Status.OK)); + associationToInsertLine.incrementCount(CollectionUtils.nonNullList(entry.getValue()).size()); + } + } + } + + runBackendStepOutput.setRecords(outputRecords); + this.rowsProcessed += records.size(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void handleBulkLoad(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List records, QTableMetaData table) throws QException + { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -209,7 +465,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { - AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); + WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, records, true); @@ -485,11 +741,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep recordsProcessedLine.withPluralFutureMessage("records were"); recordsProcessedLine.withPluralPastMessage("records were"); - String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; - okSummary.setSingularFutureMessage(tableLabel + " record will be inserted" + noWarningsSuffix + "."); - okSummary.setPluralFutureMessage(tableLabel + " records will be inserted" + noWarningsSuffix + "."); - okSummary.setSingularPastMessage(tableLabel + " record was inserted" + noWarningsSuffix + "."); - okSummary.setPluralPastMessage(tableLabel + " records were inserted" + noWarningsSuffix + "."); + boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepOutput.getValueBoolean("isBulkEdit")); + String action = isBulkEdit ? "updated" : "inserted"; + String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; + okSummary.setSingularFutureMessage(tableLabel + " record will be " + action + noWarningsSuffix + "."); + okSummary.setPluralFutureMessage(tableLabel + " records will be " + action + noWarningsSuffix + "."); + okSummary.setSingularPastMessage(tableLabel + " record was " + action + noWarningsSuffix + "."); + okSummary.setPluralPastMessage(tableLabel + " records were " + action + noWarningsSuffix + "."); okSummary.pickMessage(isForResultScreen); okSummary.addSelfToListIfAnyCount(rs); @@ -502,10 +760,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep String associationLabel = associationTable.getLabel(); ProcessSummaryLine line = entry.getValue(); - line.setSingularFutureMessage(associationLabel + " record will be inserted."); - line.setPluralFutureMessage(associationLabel + " records will be inserted."); - line.setSingularPastMessage(associationLabel + " record was inserted."); - line.setPluralPastMessage(associationLabel + " records were inserted."); + line.setSingularFutureMessage(associationLabel + " record will be " + action + "."); + line.setPluralFutureMessage(associationLabel + " records will be " + action + "."); + line.setSingularPastMessage(associationLabel + " record was " + action + "."); + line.setPluralPastMessage(associationLabel + " records were " + action + "."); line.pickMessage(isForResultScreen); line.addSelfToListIfAnyCount(rs); } @@ -518,8 +776,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values" - + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") - + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) + + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") + + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) .withSingularFutureMessage(" record will not be") .withPluralFutureMessage(" records will not be") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java index 71f0edcc..cc9eb9b0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java @@ -54,7 +54,7 @@ public class BulkLoadMappingSuggester /*************************************************************************** ** ***************************************************************************/ - public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List headerRow) + public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List headerRow, boolean isBulkEdit) { massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>(); for(int i = 0; i < headerRow.size(); i++) @@ -90,6 +90,7 @@ public class BulkLoadMappingSuggester .withVersion("v1") .withLayout(layout) .withHasHeaderRow(true) + .withIsBulkEdit(isBulkEdit) .withFieldList(fieldList); return (bulkLoadProfile); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java index 26575be3..77114b1c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java @@ -25,12 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.bulk.TableKeyFieldsPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; @@ -44,12 +51,16 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class BulkLoadTableStructureBuilder { + private static final QLogger LOG = QLogger.getLogger(BulkLoadTableStructureBuilder.class); + + + /*************************************************************************** ** ***************************************************************************/ public static BulkLoadTableStructure buildTableStructure(String tableName) { - return (buildTableStructure(tableName, null, null)); + return (buildTableStructure(tableName, null, null, false)); } @@ -57,13 +68,24 @@ public class BulkLoadTableStructureBuilder /*************************************************************************** ** ***************************************************************************/ - private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) + public static BulkLoadTableStructure buildTableStructure(String tableName, Boolean isBulkEdit) + { + return (buildTableStructure(tableName, null, null, isBulkEdit)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath, Boolean isBulkEdit) { QTableMetaData table = QContext.getQInstance().getTable(tableName); BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); tableStructure.setTableName(tableName); tableStructure.setLabel(table.getLabel()); + tableStructure.setIsBulkEdit(isBulkEdit); Set associationJoinFieldNamesToExclude = new HashSet<>(); @@ -119,6 +141,30 @@ public class BulkLoadTableStructureBuilder } } + //////////////////////////////////////////////////////// + // for bulk edit, users can use the primary key field // + //////////////////////////////////////////////////////// + if(isBulkEdit) + { + fields.add(table.getField(table.getPrimaryKeyField())); + + ////////////////////////////////////////////////////////////////////// + // also make available what key fields are available for this table // + ////////////////////////////////////////////////////////////////////// + try + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput() + .withPossibleValueSourceName("tableKeyFields") + .withPathParamMap(Map.of("processName", tableName + ".bulkEditWithFile")); + List> search = new TableKeyFieldsPossibleValueSource().search(input); + tableStructure.setPossibleKeyFields(new ArrayList<>(search.stream().map(QPossibleValue::getId).toList())); + } + catch(QException qe) + { + LOG.warn("Unable to retrieve possible key fields for table [" + tableName + "]", qe); + } + } + fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), ""))); for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) @@ -131,8 +177,8 @@ public class BulkLoadTableStructureBuilder { String nextLevelPath = (StringUtils.hasContent(parentAssociationPath) ? parentAssociationPath + "." : "") - + (association != null ? association.getName() : ""); - BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, nextLevelPath); + + (association != null ? association.getName() : ""); + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, nextLevelPath, isBulkEdit); tableStructure.addAssociation(associatedStructure); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java index 2fe07b3c..493615d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java @@ -37,6 +37,8 @@ public class BulkLoadProfile implements Serializable private Boolean hasHeaderRow; private String layout; private String version; + private Boolean isBulkEdit; + private String keyFields; @@ -132,6 +134,7 @@ public class BulkLoadProfile implements Serializable } + /******************************************************************************* ** Getter for version *******************************************************************************/ @@ -162,4 +165,65 @@ public class BulkLoadProfile implements Serializable } + + /******************************************************************************* + ** Getter for isBulkEdit + *******************************************************************************/ + public Boolean getIsBulkEdit() + { + return (this.isBulkEdit); + } + + + + /******************************************************************************* + ** Setter for isBulkEdit + *******************************************************************************/ + public void setIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + } + + + + /******************************************************************************* + ** Fluent setter for isBulkEdit + *******************************************************************************/ + public BulkLoadProfile withIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + return (this); + } + + + + /******************************************************************************* + ** Getter for keyFields + *******************************************************************************/ + public String getKeyFields() + { + return (this.keyFields); + } + + + + /******************************************************************************* + ** Setter for keyFields + *******************************************************************************/ + public void setKeyFields(String keyFields) + { + this.keyFields = keyFields; + } + + + + /******************************************************************************* + ** Fluent setter for keyFields + *******************************************************************************/ + public BulkLoadProfile withKeyFields(String keyFields) + { + this.keyFields = keyFields; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java index f7ce0f6c..17acc4c1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java @@ -35,6 +35,7 @@ public class BulkLoadProfileField private Integer columnIndex; private String headerName; private Serializable defaultValue; + private Boolean clearIfEmpty; private Boolean doValueMapping; private Map valueMappings; @@ -194,6 +195,7 @@ public class BulkLoadProfileField } + /******************************************************************************* ** Getter for headerName *******************************************************************************/ @@ -224,4 +226,34 @@ public class BulkLoadProfileField } + + /******************************************************************************* + ** Getter for clearIfEmpty + *******************************************************************************/ + public Boolean getClearIfEmpty() + { + return (this.clearIfEmpty); + } + + + + /******************************************************************************* + ** Setter for clearIfEmpty + *******************************************************************************/ + public void setClearIfEmpty(Boolean clearIfEmpty) + { + this.clearIfEmpty = clearIfEmpty; + } + + + + /******************************************************************************* + ** Fluent setter for clearIfEmpty + *******************************************************************************/ + public BulkLoadProfileField withClearIfEmpty(Boolean clearIfEmpty) + { + this.clearIfEmpty = clearIfEmpty; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java index db55198f..ff4e729b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java @@ -35,9 +35,12 @@ public class BulkLoadTableStructure implements Serializable private boolean isMain; private boolean isMany; - private String tableName; - private String label; - private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild + private String tableName; + private String label; + private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild + private Boolean isBulkEdit; + private String keyFields; + private ArrayList possibleKeyFields; private ArrayList fields; // mmm, not marked as serializable (at this time) - is okay? private ArrayList associations; @@ -272,4 +275,98 @@ public class BulkLoadTableStructure implements Serializable } this.associations.add(association); } + + + + /******************************************************************************* + ** Getter for isBulkEdit + *******************************************************************************/ + public Boolean getIsBulkEdit() + { + return (this.isBulkEdit); + } + + + + /******************************************************************************* + ** Setter for isBulkEdit + *******************************************************************************/ + public void setIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + } + + + + /******************************************************************************* + ** Fluent setter for isBulkEdit + *******************************************************************************/ + public BulkLoadTableStructure withIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + return (this); + } + + + + /******************************************************************************* + ** Getter for keyFields + *******************************************************************************/ + public String getKeyFields() + { + return (this.keyFields); + } + + + + /******************************************************************************* + ** Setter for keyFields + *******************************************************************************/ + public void setKeyFields(String keyFields) + { + this.keyFields = keyFields; + } + + + + /******************************************************************************* + ** Fluent setter for keyFields + *******************************************************************************/ + public BulkLoadTableStructure withKeyFields(String keyFields) + { + this.keyFields = keyFields; + return (this); + } + + + + /******************************************************************************* + ** Getter for possibleKeyFields + *******************************************************************************/ + public ArrayList getPossibleKeyFields() + { + return (this.possibleKeyFields); + } + + + + /******************************************************************************* + ** Setter for possibleKeyFields + *******************************************************************************/ + public void setPossibleKeyFields(ArrayList possibleKeyFields) + { + this.possibleKeyFields = possibleKeyFields; + } + + + + /******************************************************************************* + ** Fluent setter for possibleKeyFields + *******************************************************************************/ + public BulkLoadTableStructure withPossibleKeyFields(ArrayList possibleKeyFields) + { + this.possibleKeyFields = possibleKeyFields; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java index c1bdaa41..a6e6c3b5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java @@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import org.apache.commons.lang.BooleanUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -102,13 +103,29 @@ public class QuerySavedBulkLoadProfileProcess implements BackendStep } else { - String tableName = runBackendStepInput.getValueString("tableName"); + String tableName = runBackendStepInput.getValueString("tableName"); + boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit")); QueryInput input = new QueryInput(); input.setTableName(SavedBulkLoadProfile.TABLE_NAME); - input.setFilter(new QQueryFilter() + + QQueryFilter filter = new QQueryFilter() .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)) - .withOrderBy(new QFilterOrderBy("label"))); + .withOrderBy(new QFilterOrderBy("label")); + + ///////////////////////////////////////////////////////////////////// + // account for nulls here, so if is bulk edit, only look for true, // + // otherwise look for nulls or not equal to true // + ///////////////////////////////////////////////////////////////////// + if(isBulkEdit) + { + filter.withCriteria(new QFilterCriteria("isBulkEdit", QCriteriaOperator.EQUALS, true)); + } + else + { + filter.withCriteria(new QFilterCriteria("isBulkEdit", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, true)); + } + input.setFilter(filter); QueryOutput output = new QueryAction().execute(input); runBackendStepOutput.setRecords(output.getRecords()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java index 0d8f33d4..5a2e3eab 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java @@ -50,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.commons.lang.BooleanUtils; /******************************************************************************* @@ -87,9 +88,10 @@ public class StoreSavedBulkLoadProfileProcess implements BackendStep try { - String userId = QContext.getQSession().getUser().getIdReference(); - String tableName = runBackendStepInput.getValueString("tableName"); - String label = runBackendStepInput.getValueString("label"); + String userId = QContext.getQSession().getUser().getIdReference(); + String tableName = runBackendStepInput.getValueString("tableName"); + String label = runBackendStepInput.getValueString("label"); + Boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit")); String mappingJson = processMappingJson(runBackendStepInput.getValueString("mappingJson")); @@ -98,6 +100,7 @@ public class StoreSavedBulkLoadProfileProcess implements BackendStep .withValue("mappingJson", mappingJson) .withValue("label", label) .withValue("tableName", tableName) + .withValue("isBulkEdit", isBulkEdit) .withValue("userId", userId); List savedBulkLoadProfileList; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java index 0eb26407..1d49d820 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java @@ -250,7 +250,7 @@ class MetaDataActionTest extends BaseTest // with several permissions set, we should see some things, and they should have permissions turned on // ///////////////////////////////////////////////////////////////////////////////////////////////////////// assertEquals(Set.of("person"), result.getTables().keySet()); - assertEquals(Set.of("increaseBirthdate", "runShapesPersonReport", "person.bulkInsert", "person.bulkEdit", "person.bulkDelete"), result.getProcesses().keySet()); + assertEquals(Set.of("increaseBirthdate", "runShapesPersonReport", "person.bulkInsert", "person.bulkEdit", "person.bulkEditWithFile", "person.bulkDelete"), result.getProcesses().keySet()); assertEquals(Set.of("shapesPersonReport", "personJoinShapeReport", "simplePersonReport"), result.getReports().keySet()); assertEquals(Set.of("PersonsByCreateDateBarChart"), result.getWidgets().keySet()); @@ -286,7 +286,7 @@ class MetaDataActionTest extends BaseTest assertEquals(Set.of("person", "personFile", "personMemory"), result.getTables().keySet()); - assertEquals(Set.of("increaseBirthdate", "personFile.bulkInsert", "personFile.bulkEdit", "personFile.bulkDelete", "personMemory.bulkInsert", "personMemory.bulkEdit", "personMemory.bulkDelete"), result.getProcesses().keySet()); + assertEquals(Set.of("increaseBirthdate", "personFile.bulkInsert", "personFile.bulkEdit", "personFile.bulkEditWithFile", "personFile.bulkDelete", "personMemory.bulkInsert", "personMemory.bulkEdit", "personMemory.bulkEditWithFile", "personMemory.bulkDelete"), result.getProcesses().keySet()); assertEquals(Set.of(), result.getReports().keySet()); assertEquals(Set.of(), result.getWidgets().keySet()); @@ -333,7 +333,7 @@ class MetaDataActionTest extends BaseTest MetaDataOutput result = new MetaDataAction().execute(new MetaDataInput()); assertEquals(Set.of("person", "personFile", "personMemory"), result.getTables().keySet()); - assertEquals(Set.of("increaseBirthdate", "personFile.bulkInsert", "personFile.bulkEdit", "personMemory.bulkDelete"), result.getProcesses().keySet()); + assertEquals(Set.of("increaseBirthdate", "personFile.bulkInsert", "personFile.bulkEdit", "personFile.bulkEditWithFile", "personMemory.bulkDelete"), result.getProcesses().keySet()); assertEquals(Set.of(), result.getReports().keySet()); assertEquals(Set.of(), result.getWidgets().keySet()); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/editwithfile/BulkEditWithFileFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/editwithfile/BulkEditWithFileFullProcessTest.java new file mode 100644 index 00000000..6fda5077 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/editwithfile/BulkEditWithFileFullProcessTest.java @@ -0,0 +1,519 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.editwithfile; + + +import java.io.OutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertFullProcessTest; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for full bulk insert process + *******************************************************************************/ +class BulkEditWithFileFullProcessTest extends BaseTest +{ + private static final String defaultEmail = "noone@kingsrook.com"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow1() + { + return (""" + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","Jehn","Doe","1980-01-01","john@doe.com","Missouri",24 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow2() + { + return (""" + "2","2021-10-26 14:39:37","2021-10-26 14:39:37","Jyne","Doe","1981-01-01","john@doe.com","Illinois", + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvHeaderUsingLabels() + { + return (""" + "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email","Home State",noOfShoes + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws Exception + { + ///////////////////////////////////////////// + // use the bulk insert test to insert data // + ///////////////////////////////////////////// + new BulkInsertFullProcessTest().test(); + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isNotEmpty(); + + ///////////////////////////////////////////////////////// + // start the process - expect to go to the upload step // + ///////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.addValue("keyFields", "id"); + runProcessInput.addValue("isBulkEdit", "true"); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload"); + + ////////////////////////// + // continue post-upload // + ////////////////////////// + runProcessOutput = continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(2)); + assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); + assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); + + ////////////////////////////////////////////////////// + // assert about the suggested mapping that was done // + ////////////////////////////////////////////////////// + Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile"); + assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class); + assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5); + assertEquals("id", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName()); + assertEquals(0, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex()); + assertEquals("firstName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getFieldName()); + assertEquals(3, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getColumnIndex()); + assertEquals("lastName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getFieldName()); + assertEquals(4, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getColumnIndex()); + assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(3).getFieldName()); + assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(3).getColumnIndex()); + + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); + + //////////////////////////////// + // continue post file-mapping // + //////////////////////////////// + runProcessOutput = continueProcessPostFileMapping(runProcessInput); + Serializable valueMappingField = runProcessOutput.getValue("valueMappingField"); + assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class); + assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName()); + assertEquals(List.of("Missouri", "Illinois"), runProcessOutput.getValue("fileValues")); + assertEquals(List.of("homeStateId"), runProcessOutput.getValue("fieldNamesToDoValueMapping")); + assertEquals(Map.of(1, "IL"), runProcessOutput.getValue("mappedValueLabels")); + assertEquals(0, runProcessOutput.getValue("valueMappingFieldIndex")); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("valueMapping"); + + ///////////////////////////////// + // continue post value-mapping // + ///////////////////////////////// + runProcessOutput = continueProcessPostValueMapping(runProcessInput); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + + ///////////////////////////////// + // continue post review screen // + ///////////////////////////////// + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + ProcessSummaryLineInterface okLine = ProcessSummaryAssert.assertThat(runProcessOutput) + .hasLineWithMessageContaining("Person Memory records were edited") + .hasStatus(Status.OK) + .hasCount(2) + .getLine(); + assertEquals(List.of(1, 2), ((ProcessSummaryLine) okLine).getPrimaryKeys()); + + //////////////////////////////////// + // query for the inserted records // + //////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + assertEquals("Jehn", records.get(0).getValueString("firstName")); + assertEquals("Jyne", records.get(1).getValueString("firstName")); + + assertNotNull(records.get(0).getValue("id")); + assertNotNull(records.get(1).getValue("id")); + assertEquals(1, records.get(0).getValue("id")); + assertEquals(2, records.get(1).getValue("id")); + + assertEquals(2, records.get(0).getValueInteger("homeStateId")); + assertEquals(1, records.get(1).getValueInteger("homeStateId")); + + assertEquals(defaultEmail, records.get(0).getValueString("email")); + assertEquals(defaultEmail, records.get(1).getValueString("email")); + + assertEquals(24, records.get(0).getValueInteger("noOfShoes")); + assertNull(records.get(1).getValue("noOfShoes")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSummaryLinePrimaryKeys() throws Exception + { + ///////////////////////////////////////////// + // use the bulk insert test to insert data // + ///////////////////////////////////////////// + new BulkInsertFullProcessTest().testSummaryLinePrimaryKeys(); + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isNotEmpty(); + + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(PersonWarnOrErrorCustomizer.class)); + + ///////////////////////////////////////////////////////// + // start the process - expect to go to the upload step // + ///////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + continueProcessPostUpload(runProcessInput, processUUID, simulateFileUploadForWarningCase()); + continueProcessPostFileMapping(runProcessInput); + continueProcessPostValueMapping(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); + + ProcessSummaryLineInterface okLine = ProcessSummaryAssert.assertThat(runProcessOutput) + .hasLineWithMessageContaining("Person Memory records were edited") + .hasStatus(Status.OK) + .hasCount(4) + .getLine(); + assertEquals(List.of(1, 2, 3, 4), ((ProcessSummaryLine) okLine).getPrimaryKeys()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSummaryLineErrors() throws Exception + { + ///////////////////////////////////////////// + // use the bulk insert test to insert data // + ///////////////////////////////////////////// + new BulkInsertFullProcessTest().testSummaryLineErrors(); + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isNotEmpty(); + + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(PersonWarnOrErrorCustomizer.class)); + + ///////////////////////////////////////////////////////// + // start the process - expect to go to the upload step // + ///////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + continueProcessPostUpload(runProcessInput, processUUID, simulateFileUploadForErrorCase()); + continueProcessPostFileMapping(runProcessInput); + continueProcessPostValueMapping(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); + + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Person Memory record was edited.").hasStatus(Status.OK).hasCount(1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneRow() throws Exception + { + ///////////////////////////////////////////// + // use the bulk insert test to insert data // + ///////////////////////////////////////////// + new BulkInsertFullProcessTest().testSummaryLineErrors(); + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isNotEmpty(); + + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(1)); + continueProcessPostFileMapping(runProcessInput); + continueProcessPostValueMapping(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // all that just so we can make sure this message is right (because it was wrong when we first wrote it, lol) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Person Memory record was edited.").hasStatus(Status.OK).hasCount(1); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostReviewScreen(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("review"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostValueMapping(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setStartAfterStep("valueMapping"); + runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); + addProfileToRunProcessInput(runProcessInput); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostFileMapping(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("fileMapping"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostUpload(RunProcessInput runProcessInput, String processUUID, StorageInput storageInput) throws QException + { + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("upload"); + runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static StorageInput simulateFileUpload(int noOfRows) throws Exception + { + String storageReference = UUID.randomUUID() + ".csv"; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); + try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) + { + outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + (noOfRows == 2 ? getPersonCsvRow2() : "")).getBytes()); + } + return storageInput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static StorageInput simulateFileUploadForWarningCase() throws Exception + { + String storageReference = UUID.randomUUID() + ".csv"; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); + try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) + { + outputStream.write((getPersonCsvHeaderUsingLabels() + """ + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42 + "2","2021-10-26 14:39:37","2021-10-26 14:39:37","Tornado warning","Doe","1980-01-01","john@doe.com","Missouri",42 + "3","2021-10-26 14:39:37","2021-10-26 14:39:37","Tornado warning","Doey","1980-01-01","john@doe.com","Missouri",42 + "4","2021-10-26 14:39:37","2021-10-26 14:39:37","Hurricane warning","Doe","1980-01-01","john@doe.com","Missouri",42 + """).getBytes()); + } + return storageInput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static StorageInput simulateFileUploadForErrorCase() throws Exception + { + String storageReference = UUID.randomUUID() + ".csv"; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); + try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) + { + outputStream.write((getPersonCsvHeaderUsingLabels() + """ + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42 + "2","2021-10-26 14:39:37","2021-10-26 14:39:37","not-pre-Error plane","Doe","1980-01-01","john@doe.com","Missouri",42 + "3","2021-10-26 14:39:37","2021-10-26 14:39:37","Error purifier","Doe","1980-01-01","john@doe.com","Missouri",42 + """).getBytes()); + } + return storageInput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput startProcess(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkEditWithFile"); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + runProcessInput.addValue("isBulkEdit", "true"); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addProfileToRunProcessInput(RunProcessInput input) + { + input.addValue("version", "v1"); + input.addValue("layout", "FLAT"); + input.addValue("isBulkEdit", "true"); + input.addValue("keyFields", "id"); + input.addValue("hasHeaderRow", "true"); + input.addValue("fieldListJSON", JsonUtils.toJson(List.of( + new BulkLoadProfileField().withFieldName("id").withColumnIndex(0), + new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), + new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), + new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), + new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), + new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) + ))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class PersonWarnOrErrorCustomizer implements TableCustomizerInterface + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview) + { + return AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + for(QRecord record : records) + { + if(record.getValueString("firstName").toLowerCase().contains("warn")) + { + record.addWarning(new QWarningMessage(record.getValueString("firstName"))); + } + else if(record.getValueString("firstName").toLowerCase().contains("error")) + { + if(isPreview && record.getValueString("firstName").toLowerCase().contains("not-pre-error")) + { + continue; + } + + record.addError(new BadInputStatusMessage(record.getValueString("firstName"))); + } + } + return records; + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index cb11631b..f6e4d97c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -67,7 +67,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* ** Unit test for full bulk insert process *******************************************************************************/ -class BulkInsertFullProcessTest extends BaseTest +public class BulkInsertFullProcessTest extends BaseTest { private static final String defaultEmail = "noone@kingsrook.com"; @@ -125,7 +125,7 @@ class BulkInsertFullProcessTest extends BaseTest ** *******************************************************************************/ @Test - void test() throws Exception + public void test() throws Exception { assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); @@ -224,7 +224,7 @@ class BulkInsertFullProcessTest extends BaseTest ** *******************************************************************************/ @Test - void testSummaryLinePrimaryKeys() throws Exception + public void testSummaryLinePrimaryKeys() throws Exception { assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) @@ -267,7 +267,7 @@ class BulkInsertFullProcessTest extends BaseTest ** *******************************************************************************/ @Test - void testSummaryLineErrors() throws Exception + public void testSummaryLineErrors() throws Exception { assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) @@ -304,7 +304,7 @@ class BulkInsertFullProcessTest extends BaseTest ** *******************************************************************************/ @Test - void testOneRow() throws Exception + public void testOneRow() throws Exception { /////////////////////////////////////// // make sure table is empty to start // @@ -514,4 +514,4 @@ class BulkInsertFullProcessTest extends BaseTest return records; } } -} \ No newline at end of file +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java index a54e08ef..21ad3a09 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java @@ -38,7 +38,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* - ** Unit test for BulkLoadMappingSuggester + ** Unit test for BulkLoadMappingSuggester *******************************************************************************/ class BulkLoadMappingSuggesterTest extends BaseTest { @@ -52,7 +52,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY); List headerRow = List.of("Id", "First Name", "lastname", "email", "homestate"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("FLAT", bulkLoadProfile.getLayout()); assertNull(getFieldByName(bulkLoadProfile, "id")); @@ -73,7 +73,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "shipto name", "sku", "quantity"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("TALL", bulkLoadProfile.getLayout()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); @@ -93,7 +93,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("TALL", bulkLoadProfile.getLayout()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); @@ -120,7 +120,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); @@ -136,7 +136,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex()); @@ -152,7 +152,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "ship to name", "address", "address 2"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); @@ -177,7 +177,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("WIDE", bulkLoadProfile.getLayout()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); @@ -206,4 +206,4 @@ class BulkLoadMappingSuggesterTest extends BaseTest .findFirst().orElse(null)); } -} \ No newline at end of file +} diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index ad6ccb23..fe7db297 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -1940,6 +1940,8 @@ public class QJavalinImplementation input.setSearchTerm(searchTerm); input.setDefaultQueryFilter(defaultFilter); input.setOtherValues(otherValues); + input.setPathParamMap(context.pathParamMap()); + input.setQueryParamMap(context.queryParamMap()); if(StringUtils.hasContent(ids)) { diff --git a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml index 61b53975..2d2b8fa4 100644 --- a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml +++ b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml @@ -1909,6 +1909,13 @@ paths: name: "person.bulkDelete" stepFlow: "LINEAR" tableName: "person" + person.bulkEditWithFile: + hasPermission: true + isHidden: true + label: "Person Bulk Edit With File" + name: "person.bulkEditWithFile" + stepFlow: "LINEAR" + tableName: "person" tables: person: capabilities: