From 0a236b8c366103e95aa7845ecb358cb7916a03be Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Thu, 17 Jul 2025 13:21:49 -0500 Subject: [PATCH] initial checkin of support of bulk load with file --- ...lkTableActionProcessPermissionChecker.java | 2 +- .../core/instances/QInstanceEnricher.java | 143 ++++++++- .../SearchPossibleValueSourceInput.java | 76 ++++- .../TableKeyFieldsPossibleValueSource.java | 153 ++++++++++ .../SavedBulkLoadProfile.java | 34 ++- .../SavedBulkLoadProfileMetaDataProvider.java | 2 +- .../bulk/edit/BulkEditLoadStep.java | 94 +++++- .../BulkInsertPrepareFileMappingStep.java | 17 +- .../BulkInsertPrepareFileUploadStep.java | 15 +- .../bulk/insert/BulkInsertStepUtils.java | 7 +- .../bulk/insert/BulkInsertTransformStep.java | 288 +++++++++++++++++- .../mapping/BulkLoadMappingSuggester.java | 3 +- .../BulkLoadTableStructureBuilder.java | 54 +++- .../bulk/insert/model/BulkLoadProfile.java | 64 ++++ .../insert/model/BulkLoadProfileField.java | 32 ++ .../insert/model/BulkLoadTableStructure.java | 103 ++++++- .../QuerySavedBulkLoadProfileProcess.java | 23 +- .../StoreSavedBulkLoadProfileProcess.java | 9 +- .../mapping/BulkLoadMappingSuggesterTest.java | 18 +- .../javalin/QJavalinImplementation.java | 2 + 20 files changed, 1073 insertions(+), 66 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/bulk/TableKeyFieldsPossibleValueSource.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java index 5c9f0e14..9cdefa74 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java @@ -57,7 +57,7 @@ public class BulkTableActionProcessPermissionChecker implements CustomPermission switch(bulkActionName) { case "bulkInsert" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.INSERT); - case "bulkEdit" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.EDIT); + case "bulkEdit", "bulkEditWithFile" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.EDIT); case "bulkDelete" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.DELETE); default -> LOG.warn("Unexpected bulk action name when checking permissions for process: " + processName); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 6895a258..384a4907 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvide import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.bulk.TableKeyFieldsPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -857,6 +858,8 @@ public class QInstanceEnricher *******************************************************************************/ private void defineTableBulkProcesses(QInstance qInstance) { + qInstance.addPossibleValueSource(defineTableKeyFieldsPossibleValueSource()); + for(QTableMetaData table : qInstance.getTables().values()) { if(table.getFields() == null) @@ -880,6 +883,12 @@ public class QInstanceEnricher defineTableBulkEdit(qInstance, table, bulkEditProcessName); } + String bulkEditWithFileProcessName = table.getName() + ".bulkEditWithFile"; + if(qInstance.getProcess(bulkEditWithFileProcessName) == null) + { + defineTableBulkEditWithFile(qInstance, table, bulkEditWithFileProcessName); + } + String bulkDeleteProcessName = table.getName() + ".bulkDelete"; if(qInstance.getProcess(bulkDeleteProcessName) == null) { @@ -984,16 +993,16 @@ public class QInstanceEnricher .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class)); int i = 0; - process.addStep(i++, prepareFileUploadStep); - process.addStep(i++, uploadScreen); + process.withStep(i++, prepareFileUploadStep); + process.withStep(i++, uploadScreen); - process.addStep(i++, prepareFileMappingStep); - process.addStep(i++, fileMappingScreen); - process.addStep(i++, receiveFileMappingStep); + process.withStep(i++, prepareFileMappingStep); + process.withStep(i++, fileMappingScreen); + process.withStep(i++, receiveFileMappingStep); - process.addStep(i++, prepareValueMappingStep); - process.addStep(i++, valueMappingScreen); - process.addStep(i++, receiveValueMappingStep); + process.withStep(i++, prepareValueMappingStep); + process.withStep(i++, valueMappingScreen); + process.withStep(i++, receiveValueMappingStep); process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields); @@ -1023,7 +1032,7 @@ public class QInstanceEnricher values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE); QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( - ExtractViaQueryStep.class, + BulkInsertExtractStep.class, BulkEditTransformStep.class, BulkEditLoadStep.class, values @@ -1060,6 +1069,122 @@ public class QInstanceEnricher + /******************************************************************************* + ** + *******************************************************************************/ + public void defineTableBulkEditWithFile(QInstance qInstance, QTableMetaData table, String processName) + { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "This is a preview of the records that will be updated."); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + BulkInsertExtractStep.class, + BulkInsertTransformStep.class, + BulkEditLoadStep.class, + values + ) + .withName(processName) + .withLabel(table.getLabel() + " Bulk Edit With File") + .withTableName(table.getName()) + .withIsHidden(true) + .withPermissionRules(qInstance.getDefaultPermissionRules().clone() + .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class))); + + List editableFields = table.getFields().values().stream() + .filter(QFieldMetaData::getIsEditable) + .filter(f -> !f.getType().equals(QFieldType.BLOB)) + .toList(); + + QBackendStepMetaData prepareFileUploadStep = new QBackendStepMetaData() + .withName("prepareFileUpload") + .withCode(new QCodeReference(BulkInsertPrepareFileUploadStep.class)); + + QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() + .withName("upload") + .withLabel("Upload File") + .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB) + .withFieldAdornment(FileUploadAdornment.newFieldAdornment() + .withValue(FileUploadAdornment.formatDragAndDrop()) + .withValue(FileUploadAdornment.widthFull())) + .withLabel(table.getLabel() + " File") + .withIsRequired(true)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); + + QBackendStepMetaData prepareFileMappingStep = new QBackendStepMetaData() + .withName("prepareFileMapping") + .withCode(new QCodeReference(BulkInsertPrepareFileMappingStep.class)); + + QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() + .withName("fileMapping") + .withLabel("File Mapping") + .withBackStepName("prepareFileUpload") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)) + .withFormField(new QFieldMetaData("hasHeaderRow", QFieldType.BOOLEAN)) + .withFormField(new QFieldMetaData("layout", QFieldType.STRING)) // is actually PVS, but, this field is only added to help support helpContent, so :shrug: + .withFormField(new QFieldMetaData("tableKeyFields", QFieldType.STRING).withPossibleValueSourceName(TableKeyFieldsPossibleValueSource.NAME)); + + QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData() + .withName("receiveFileMapping") + .withCode(new QCodeReference(BulkInsertReceiveFileMappingStep.class)); + + QBackendStepMetaData prepareValueMappingStep = new QBackendStepMetaData() + .withName("prepareValueMapping") + .withCode(new QCodeReference(BulkInsertPrepareValueMappingStep.class)); + + QFrontendStepMetaData valueMappingScreen = new QFrontendStepMetaData() + .withName("valueMapping") + .withLabel("Value Mapping") + .withBackStepName("prepareFileMapping") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM)); + + QBackendStepMetaData receiveValueMappingStep = new QBackendStepMetaData() + .withName("receiveValueMapping") + .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class)); + + int i = 0; + process.withStep(i++, prepareFileUploadStep); + process.withStep(i++, uploadScreen); + + process.withStep(i++, prepareFileMappingStep); + process.withStep(i++, fileMappingScreen); + process.withStep(i++, receiveFileMappingStep); + + process.withStep(i++, prepareValueMappingStep); + process.withStep(i++, valueMappingScreen); + process.withStep(i++, receiveValueMappingStep); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields); + + ////////////////////////////////////////////////////////////////////////////////////////// + // put the bulk-load profile form (e.g., for saving it) on the review & result screens) // + ////////////////////////////////////////////////////////////////////////////////////////// + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW) + .withBackStepName("prepareFileMapping") + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_RESULT) + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + + qInstance.addProcess(process); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QPossibleValueSource defineTableKeyFieldsPossibleValueSource() + { + return (new QPossibleValueSource() + .withName(TableKeyFieldsPossibleValueSource.NAME) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(TableKeyFieldsPossibleValueSource.class))); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java index 8976a176..64b446eb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.values; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -34,11 +35,13 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; *******************************************************************************/ public class SearchPossibleValueSourceInput extends AbstractActionInput implements Cloneable { - private String possibleValueSourceName; - private QQueryFilter defaultQueryFilter; - private String searchTerm; - private List idList; - private List labelList; + private String possibleValueSourceName; + private QQueryFilter defaultQueryFilter; + private String searchTerm; + private List idList; + private List labelList; + private Map pathParamMap; + private Map> queryParamMap; private Integer skip = 0; private Integer limit = 250; @@ -284,6 +287,7 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen } + /******************************************************************************* ** Getter for labelList *******************************************************************************/ @@ -313,4 +317,66 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen return (this); } + + + /******************************************************************************* + ** Getter for pathParamMap + *******************************************************************************/ + public Map getPathParamMap() + { + return (this.pathParamMap); + } + + + + /******************************************************************************* + ** Setter for pathParamMap + *******************************************************************************/ + public void setPathParamMap(Map pathParamMap) + { + this.pathParamMap = pathParamMap; + } + + + + /******************************************************************************* + ** Fluent setter for pathParamMap + *******************************************************************************/ + public SearchPossibleValueSourceInput withPathParamMap(Map pathParamMap) + { + this.pathParamMap = pathParamMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for queryParamMap + *******************************************************************************/ + public Map> getQueryParamMap() + { + return (this.queryParamMap); + } + + + + /******************************************************************************* + ** Setter for queryParamMap + *******************************************************************************/ + public void setQueryParamMap(Map> queryParamMap) + { + this.queryParamMap = queryParamMap; + } + + + + /******************************************************************************* + ** Fluent setter for queryParamMap + *******************************************************************************/ + public SearchPossibleValueSourceInput withQueryParamMap(Map> queryParamMap) + { + this.queryParamMap = queryParamMap; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/bulk/TableKeyFieldsPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/bulk/TableKeyFieldsPossibleValueSource.java new file mode 100644 index 00000000..45e42a84 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/bulk/TableKeyFieldsPossibleValueSource.java @@ -0,0 +1,153 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.bulk; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TableKeyFieldsPossibleValueSource implements QCustomPossibleValueProvider +{ + public static final String NAME = "tableKeyFields"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable tableAndKey) + { + QPossibleValue possibleValue = null; + + ///////////////////////////////////////////////////////////// + // keys are in the format -|| // + ///////////////////////////////////////////////////////////// + String[] keyParts = tableAndKey.toString().split("-"); + String tableName = keyParts[0]; + String key = keyParts[1]; + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table.getPrimaryKeyField().equals(key)) + { + String id = table.getPrimaryKeyField(); + String label = table.getField(table.getPrimaryKeyField()).getLabel(); + possibleValue = new QPossibleValue<>(id, label); + } + else + { + for(UniqueKey uniqueKey : table.getUniqueKeys()) + { + String potentialMatch = getIdFromUniqueKey(uniqueKey); + if(potentialMatch.equals(key)) + { + String id = potentialMatch; + String label = getLabelFromUniqueKey(table, uniqueKey); + possibleValue = new QPossibleValue<>(id, label); + break; + } + } + } + + return (possibleValue); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + List> rs = new ArrayList<>(); + if(!CollectionUtils.nonNullMap(input.getPathParamMap()).containsKey("processName") || input.getPathParamMap().get("processName") == null || input.getPathParamMap().get("processName").isEmpty()) + { + throw (new QException("Path Param of processName was not found.")); + } + + //////////////////////////////////////////////////// + // process name will be like tnt.bulkEditWithFile // + //////////////////////////////////////////////////// + String processName = input.getPathParamMap().get("processName"); + String tableName = processName.split("\\.")[0]; + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys())) + { + String id = getIdFromUniqueKey(uniqueKey); + String label = getLabelFromUniqueKey(table, uniqueKey); + if(!StringUtils.hasContent(input.getSearchTerm()) || input.getSearchTerm().equals(id)) + { + rs.add(new QPossibleValue<>(id, label)); + } + } + rs.sort(Comparator.comparing(QPossibleValue::getLabel)); + + /////////////////////////////// + // put the primary key first // + /////////////////////////////// + if(!StringUtils.hasContent(input.getSearchTerm()) || input.getSearchTerm().equals(table.getPrimaryKeyField())) + { + rs.add(0, new QPossibleValue<>(table.getPrimaryKeyField(), table.getField(table.getPrimaryKeyField()).getLabel())); + } + + return rs; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getIdFromUniqueKey(UniqueKey uniqueKey) + { + return (StringUtils.join("|", uniqueKey.getFieldNames())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getLabelFromUniqueKey(QTableMetaData tableMetaData, UniqueKey uniqueKey) + { + List fieldLabels = new ArrayList<>(uniqueKey.getFieldNames().stream().map(f -> tableMetaData.getField(f).getLabel()).toList()); + fieldLabels.sort(Comparator.naturalOrder()); + return (StringUtils.joinWithCommasAndAnd(fieldLabels)); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java index 65f07c77..092dfd85 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java @@ -60,6 +60,9 @@ public class SavedBulkLoadProfile extends QRecordEntity @QField(label = "Mapping JSON") private String mappingJson; + @QField() + private Boolean isBulkEdit; + /******************************************************************************* @@ -251,7 +254,6 @@ public class SavedBulkLoadProfile extends QRecordEntity - /******************************************************************************* ** Getter for mappingJson *******************************************************************************/ @@ -282,4 +284,34 @@ public class SavedBulkLoadProfile extends QRecordEntity } + + /******************************************************************************* + ** Getter for isBulkEdit + *******************************************************************************/ + public Boolean getIsBulkEdit() + { + return (this.isBulkEdit); + } + + + + /******************************************************************************* + ** Setter for isBulkEdit + *******************************************************************************/ + public void setIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + } + + + + /******************************************************************************* + ** Fluent setter for isBulkEdit + *******************************************************************************/ + public SavedBulkLoadProfile withIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java index 1438dd37..ddc3e062 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java @@ -113,7 +113,7 @@ public class SavedBulkLoadProfileMetaDataProvider .withFieldsFromEntity(SavedBulkLoadProfile.class) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) - .withSection(new QFieldSection("details", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "mappingJson"))) + .withSection(new QFieldSection("details", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "mappingJson", "isBulkEdit"))) .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); table.getField("mappingJson").withBehavior(SavedBulkLoadProfileJsonFieldDisplayValueFormatter.getInstance()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java index 72eaa3fb..abf9097c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java @@ -36,10 +36,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.commons.lang.BooleanUtils; import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.buildInfoSummaryLines; @@ -53,6 +55,9 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); private List infoSummaries = new ArrayList<>(); + private Serializable firstInsertedPrimaryKey = null; + private Serializable lastInsertedPrimaryKey = null; + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited"); private String tableLabel; @@ -106,7 +111,15 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar tableLabel = table.getLabel(); } - buildInfoSummaryLines(runBackendStepInput, table, infoSummaries, true); + boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit")); + if(isBulkEdit) + { + buildBulkUpdateWithFileInfoSummaryLines(runBackendStepOutput, table); + } + else + { + buildInfoSummaryLines(runBackendStepInput, table, infoSummaries, true); + } } @@ -146,4 +159,83 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar } } + + + /*************************************************************************** + ** + ***************************************************************************/ + private void buildBulkUpdateWithFileInfoSummaryLines(RunBackendStepOutput runBackendStepOutput, QTableMetaData table) + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // the transform step builds summary lines that it predicts will update successfully. // + // but those lines don't have ids, which we'd like to have (e.g., for a process trace that // + // might link to the built record). also, it's possible that there was a fail that only // + // happened in the actual update, so, basically, re-do the summary here // + ///////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep(); + ProcessSummaryLine okSummary = transformStep.okSummary; + okSummary.setCount(0); + okSummary.setPrimaryKeys(new ArrayList<>()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // but - since errors from the transform step don't even make it through to us here in the load step, // + // do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows: // + // clear out its warnings - we'll completely rebuild them here (with primary keys) // + // and add new error lines, e.g., in case of errors that only happened past the validation if possible. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup; + processSummaryWarningsAndErrorsRollup.resetWarnings(); + + List updatedRecords = runBackendStepOutput.getRecords(); + for(QRecord updatedRecord : updatedRecords) + { + Serializable primaryKey = updatedRecord.getValue(table.getPrimaryKeyField()); + if(CollectionUtils.nullSafeIsEmpty(updatedRecord.getErrors()) && primaryKey != null) + { + ///////////////////////////////////////////////////////////////////////// + // if the record had no errors, and we have a primary key for it, then // + // keep track of the range of primary keys (first and last) // + ///////////////////////////////////////////////////////////////////////// + if(firstInsertedPrimaryKey == null) + { + firstInsertedPrimaryKey = primaryKey; + } + + lastInsertedPrimaryKey = primaryKey; + + if(!CollectionUtils.nullSafeIsEmpty(updatedRecord.getWarnings())) + { + //////////////////////////////////////////////////////////////////////////// + // if there were warnings on the updated record, put it in a warning line // + //////////////////////////////////////////////////////////////////////////// + String message = updatedRecord.getWarnings().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey); + } + else + { + /////////////////////////////////////////////////////////////////////// + // if no warnings for the updated record, then put it in the OK line // + /////////////////////////////////////////////////////////////////////// + okSummary.incrementCountAndAddPrimaryKey(primaryKey); + } + } + else + { + ////////////////////////////////////////////////////////////////////// + // else if there were errors or no primary key, build an error line // + ////////////////////////////////////////////////////////////////////// + String message = "Failed to update"; + if(!CollectionUtils.nullSafeIsEmpty(updatedRecord.getErrors())) + { + ////////////////////////////////////////////////////////// + // use the error message from the record if we have one // + ////////////////////////////////////////////////////////// + message = updatedRecord.getErrors().get(0).getMessage(); + } + processSummaryWarningsAndErrorsRollup.addError(message, primaryKey); + } + } + + okSummary.pickMessage(true); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index 19e2dc79..5a3820d5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang.BooleanUtils; import org.json.JSONObject; @@ -65,9 +66,11 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep { buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); + boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit")); String tableName = runBackendStepInput.getValueString("tableName"); - BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName, isBulkEdit); runBackendStepOutput.addValue("tableStructure", tableStructure); + runBackendStepOutput.addValue("isBulkEdit", isBulkEdit); boolean needSuggestedMapping = true; if(runBackendStepOutput.getProcessState().getIsStepBack()) @@ -81,7 +84,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep { @SuppressWarnings("unchecked") List headerValues = (List) runBackendStepOutput.getValue("headerValues"); - buildSuggestedMapping(headerValues, getPrepopulatedValues(runBackendStepInput), tableStructure, runBackendStepOutput); + buildSuggestedMapping(isBulkEdit, headerValues, getPrepopulatedValues(runBackendStepInput), tableStructure, runBackendStepOutput); } } @@ -95,8 +98,8 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep String prepopulatedValuesJson = runBackendStepInput.getValueString("prepopulatedValues"); if(StringUtils.hasContent(prepopulatedValuesJson)) { - Map rs = new LinkedHashMap<>(); - JSONObject jsonObject = JsonUtils.toJSONObject(prepopulatedValuesJson); + Map rs = new LinkedHashMap<>(); + JSONObject jsonObject = JsonUtils.toJSONObject(prepopulatedValuesJson); for(String key : jsonObject.keySet()) { rs.put(key, jsonObject.optString(key, null)); @@ -112,16 +115,16 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep /*************************************************************************** ** ***************************************************************************/ - private void buildSuggestedMapping(List headerValues, Map prepopulatedValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) + private void buildSuggestedMapping(boolean isBulkEdit, List headerValues, Map prepopulatedValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) { BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); - BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); + BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues, isBulkEdit); if(CollectionUtils.nullSafeHasContents(prepopulatedValues)) { for(Map.Entry entry : prepopulatedValues.entrySet()) { - String fieldName = entry.getKey(); + String fieldName = entry.getKey(); boolean foundFieldInProfile = false; for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index 4b15b720..1149e279 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -65,10 +65,12 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep runBackendStepOutput.addValue("theFile", null); } + boolean isBulkEdit = runBackendStepInput.getProcessName().endsWith("EditWithFile"); String tableName = runBackendStepInput.getValueString("tableName"); QTableMetaData table = QContext.getQInstance().getTable(tableName); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); runBackendStepOutput.addValue("tableStructure", tableStructure); + runBackendStepOutput.addValue("isBulkEdit", isBulkEdit); List requiredFields = new ArrayList<>(); List additionalFields = new ArrayList<>(); @@ -84,6 +86,14 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep } } + ///////////////////////////////////////////// + // bulk edit allows primary key as a field // + ///////////////////////////////////////////// + if(isBulkEdit) + { + requiredFields.add(0, table.getField(table.getPrimaryKeyField())); + } + StringBuilder html; String childTableLabels = ""; @@ -96,11 +106,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep ////////////////////////////////////////////////////////////////////////////////////////////////////////////// boolean listFieldsInHelpText = false; - if(!CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) + if(isBulkEdit || !CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) { html = new StringBuilder("""

Upload either a CSV or Excel (.xlsx) file, with one row for each record you want to - insert in the ${tableLabel} table.


+ ${action} in the ${tableLabel} table.


Your file can contain any number of columns. You will be prompted to map fields from the ${tableLabel} table to columns from your file or default values for all records that @@ -204,6 +214,7 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep finishCSV(flatCSV); String htmlString = html.toString() + .replace("${action}", (isBulkEdit ? "edit" : "insert")) .replace("${tableLabel}", table.getLabel()) .replace("${childTableLabels}", childTableLabels) .replace("${flatCSV}", Base64.getEncoder().encodeToString(flatCSV.toString().getBytes(StandardCharsets.UTF_8))) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index e9c09cea..e63fb0c1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -113,6 +113,8 @@ public class BulkInsertStepUtils { String layout = runBackendStepInput.getValueString("layout"); Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow"); + String keyFields = runBackendStepInput.getValueString("keyFields"); + Boolean isBulkEdit = runBackendStepInput.getValueBoolean("isBulkEdit"); ArrayList fieldList = new ArrayList<>(); @@ -127,6 +129,7 @@ public class BulkInsertStepUtils bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); + bulkLoadProfileField.setClearIfEmpty(jsonObject.optBoolean("clearIfEmpty")); if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings")) { @@ -140,6 +143,8 @@ public class BulkInsertStepUtils } BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withIsBulkEdit(isBulkEdit) + .withKeyFields(keyFields) .withVersion(version) .withFieldList(fieldList) .withHasHeaderRow(hasHeaderRow) @@ -213,7 +218,7 @@ public class BulkInsertStepUtils { return (processTracerKeyRecordMessage); } - + return (null); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index e3979a07..17ced69e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -32,12 +33,13 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -48,6 +50,11 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; @@ -68,6 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang.BooleanUtils; +import org.json.JSONArray; +import org.json.JSONObject; /******************************************************************************* @@ -75,9 +85,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class BulkInsertTransformStep extends AbstractTransformStep { - ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + public ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); - ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") + public ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") .withDoReplaceSingletonCountLinesWithSuffixOnly(false); private ListingHash errorToExampleRowValueMap = new ListingHash<>(); @@ -190,6 +200,252 @@ public class BulkInsertTransformStep extends AbstractTransformStep QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); List records = runBackendStepInput.getRecords(); + if(BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit"))) + { + handleBulkEdit(runBackendStepInput, runBackendStepOutput, records, table); + runBackendStepOutput.addValue("isBulkEdit", true); + } + else + { + handleBulkLoad(runBackendStepInput, runBackendStepOutput, records, table); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void handleBulkEdit(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List records, QTableMetaData table) throws QException + { + /////////////////////////////////////////// + // get the key fields for this bulk edit // + /////////////////////////////////////////// + String keyFieldsString = runBackendStepInput.getValueString("keyFields"); + List keyFields = Arrays.asList(keyFieldsString.split("\\|")); + + ////////////////////////////////////////////////////////////////////////// + // if the key field is the primary key, then just look up those records // + ////////////////////////////////////////////////////////////////////////// + List nonMatchingRecords = new ArrayList<>(); + List oldRecords = new ArrayList<>(); + List recordsToUpdate = new ArrayList<>(); + if(keyFields.size() == 1 && table.getPrimaryKeyField().equals(keyFields.get(0))) + { + recordsToUpdate = records; + String primaryKeyName = table.getPrimaryKeyField(); + List primaryKeys = records.stream().map(record -> record.getValue(primaryKeyName)).toList(); + oldRecords = new QueryAction().execute(new QueryInput(table.getName()).withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeys)))).getRecords(); + + /////////////////////////////////////////// + // get a set of old records primary keys // + /////////////////////////////////////////// + Set matchedPrimaryKeys = oldRecords.stream() + .map(r -> r.getValue(table.getPrimaryKeyField())) + .collect(java.util.stream.Collectors.toSet()); + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // iterate over file records and if primary keys dont match, add to the non matchin records list // + /////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : records) + { + Serializable recordKey = record.getValue(table.getPrimaryKeyField()); + if(!matchedPrimaryKeys.contains(recordKey)) + { + nonMatchingRecords.add(record); + } + } + } + else + { + Set uniqueIds = new HashSet<>(); + List potentialRecords = new ArrayList<>(); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // if not using the primary key, then we will look up all records for each part of the unique key // + // and for each found, if all unique parts match we will add to our list of database records // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(String uniqueKeyPart : keyFields) + { + List values = records.stream().map(record -> record.getValue(uniqueKeyPart)).toList(); + for(QRecord databaseRecord : new QueryAction().execute(new QueryInput(table.getName()).withFilter(new QQueryFilter(new QFilterCriteria(uniqueKeyPart, QCriteriaOperator.IN, values)))).getRecords()) + { + if(!uniqueIds.contains(databaseRecord.getValue(table.getPrimaryKeyField()))) + { + potentialRecords.add(databaseRecord); + uniqueIds.add(databaseRecord.getValue(table.getPrimaryKeyField())); + } + } + } + + /////////////////////////////////////////////////////////////////////////////// + // now iterate over all of the potential records checking each unique fields // + /////////////////////////////////////////////////////////////////////////////// + fileRecordLoop: + for(QRecord fileRecord : records) + { + for(QRecord databaseRecord : potentialRecords) + { + boolean allMatch = true; + + for(String uniqueKeyPart : keyFields) + { + if(!Objects.equals(fileRecord.getValue(uniqueKeyPart), databaseRecord.getValue(uniqueKeyPart))) + { + allMatch = false; + } + } + + ////////////////////////////////////////////////////////////////////////////////////// + // if we get here with all matching, update the record from the file's primary key, // + // add it to the list to update, and continue looping over file records // + ////////////////////////////////////////////////////////////////////////////////////// + if(allMatch) + { + oldRecords.add(databaseRecord); + fileRecord.setValue(table.getPrimaryKeyField(), databaseRecord.getValue(table.getPrimaryKeyField())); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // iterate over the fields in the bulk load profile, if the value for that field is empty and the value // + // of 'clear if empty' is set to true, then update the record to update with the old record's value // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + JSONArray array = new JSONArray(runBackendStepInput.getValueString("fieldListJSON")); + for(int i = 0; i < array.length(); i++) + { + JSONObject jsonObject = array.getJSONObject(i); + String fieldName = jsonObject.optString("fieldName"); + boolean clearIfEmpty = jsonObject.optBoolean("clearIfEmpty"); + + if(fileRecord.getValue(fieldName) == null) + { + if(clearIfEmpty) + { + fileRecord.setValue(fieldName, null); + } + else + { + fileRecord.setValue(fieldName, databaseRecord.getValue(fieldName)); + } + } + } + + recordsToUpdate.add(fileRecord); + continue fileRecordLoop; + } + } + + /////////////////////////////////////////////////////////////////////////////////////// + // if we make it here, that means the record was not found, keep for logging warning // + /////////////////////////////////////////////////////////////////////////////////////// + nonMatchingRecords.add(fileRecord); + } + } + + for(QRecord missingRecord : CollectionUtils.nonNullList(nonMatchingRecords)) + { + String message = "Did not have a matching existing record."; + processSummaryWarningsAndErrorsRollup.addError(message, null); + addToErrorToExampleRowMap(message, missingRecord); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + UpdateInput updateInput = new UpdateInput(table.getName()); + updateInput.setInputSource(QInputSource.USER); + updateInput.setRecords(recordsToUpdate); + + ////////////////////////////////////////////////////////////////////// + // load the pre-insert customizer and set it up, if there is one // + // then we'll run it based on its WhenToRun value // + // we do this, in case it needs to, for example, adjust values that // + // are part of a unique key // + ////////////////////////////////////////////////////////////////////// + boolean didAlreadyRunCustomizer = false; + Optional preUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_UPDATE_RECORD.getRole()); + if(preUpdateCustomizer.isPresent()) + { + List recordsAfterCustomizer = preUpdateCustomizer.get().preUpdate(updateInput, records, true, Optional.of(oldRecords)); + runBackendStepInput.setRecords(recordsAfterCustomizer); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // so we used to have a comment here asking "do we care if the customizer runs both now, and in the validation below?" // + // when implementing Bulk Load V2, we were seeing that some customizers were adding errors to records, both now, and // + // when they ran below. so, at that time, we added this boolean, to track and avoid the double-run... // + // we could also imagine this being a setting on the pre-insert customizer, similar to its whenToRun attribute... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + didAlreadyRunCustomizer = true; + } + + ///////////////////////////////////////////////////////////////////////////////// + // run all validation from the insert action - in Preview mode (boolean param) // + ///////////////////////////////////////////////////////////////////////////////// + updateInput.setRecords(recordsToUpdate); + UpdateAction updateAction = new UpdateAction(); + updateAction.performValidations(updateInput, Optional.of(recordsToUpdate), didAlreadyRunCustomizer); + List validationResultRecords = updateInput.getRecords(); + + ///////////////////////////////////////////////////////////////// + // look at validation results to build process summary results // + ///////////////////////////////////////////////////////////////// + List outputRecords = new ArrayList<>(); + for(QRecord record : validationResultRecords) + { + List errorsFromAssociations = getErrorsFromAssociations(record); + if(CollectionUtils.nullSafeHasContents(errorsFromAssociations)) + { + List recordErrors = Objects.requireNonNullElseGet(record.getErrors(), () -> new ArrayList<>()); + recordErrors.addAll(errorsFromAssociations); + record.setErrors(recordErrors); + } + + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + for(QErrorMessage error : record.getErrors()) + { + if(error instanceof AbstractBulkLoadRollableValueError rollableValueError) + { + processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(rollableValueError, record); + } + else + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + addToErrorToExampleRowMap(error.getMessage(), record); + } + } + } + else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) + { + String message = record.getWarnings().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addWarning(message, null); + outputRecords.add(record); + } + else + { + okSummary.incrementCountAndAddPrimaryKey(null); + outputRecords.add(record); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + String associationName = entry.getKey(); + ProcessSummaryLine associationToInsertLine = associationsToInsertSummaries.computeIfAbsent(associationName, x -> new ProcessSummaryLine(Status.OK)); + associationToInsertLine.incrementCount(CollectionUtils.nonNullList(entry.getValue()).size()); + } + } + } + + runBackendStepOutput.setRecords(outputRecords); + this.rowsProcessed += records.size(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void handleBulkLoad(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List records, QTableMetaData table) throws QException + { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -209,7 +465,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { - AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); + WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, records, true); @@ -485,11 +741,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep recordsProcessedLine.withPluralFutureMessage("records were"); recordsProcessedLine.withPluralPastMessage("records were"); - String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; - okSummary.setSingularFutureMessage(tableLabel + " record will be inserted" + noWarningsSuffix + "."); - okSummary.setPluralFutureMessage(tableLabel + " records will be inserted" + noWarningsSuffix + "."); - okSummary.setSingularPastMessage(tableLabel + " record was inserted" + noWarningsSuffix + "."); - okSummary.setPluralPastMessage(tableLabel + " records were inserted" + noWarningsSuffix + "."); + boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepOutput.getValueBoolean("isBulkEdit")); + String action = isBulkEdit ? "updated" : "inserted"; + String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; + okSummary.setSingularFutureMessage(tableLabel + " record will be " + action + noWarningsSuffix + "."); + okSummary.setPluralFutureMessage(tableLabel + " records will be " + action + noWarningsSuffix + "."); + okSummary.setSingularPastMessage(tableLabel + " record was " + action + noWarningsSuffix + "."); + okSummary.setPluralPastMessage(tableLabel + " records were " + action + noWarningsSuffix + "."); okSummary.pickMessage(isForResultScreen); okSummary.addSelfToListIfAnyCount(rs); @@ -502,10 +760,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep String associationLabel = associationTable.getLabel(); ProcessSummaryLine line = entry.getValue(); - line.setSingularFutureMessage(associationLabel + " record will be inserted."); - line.setPluralFutureMessage(associationLabel + " records will be inserted."); - line.setSingularPastMessage(associationLabel + " record was inserted."); - line.setPluralPastMessage(associationLabel + " records were inserted."); + line.setSingularFutureMessage(associationLabel + " record will be " + action + "."); + line.setPluralFutureMessage(associationLabel + " records will be " + action + "."); + line.setSingularPastMessage(associationLabel + " record was " + action + "."); + line.setPluralPastMessage(associationLabel + " records were " + action + "."); line.pickMessage(isForResultScreen); line.addSelfToListIfAnyCount(rs); } @@ -518,8 +776,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values" - + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") - + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) + + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") + + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) .withSingularFutureMessage(" record will not be") .withPluralFutureMessage(" records will not be") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java index 71f0edcc..cc9eb9b0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java @@ -54,7 +54,7 @@ public class BulkLoadMappingSuggester /*************************************************************************** ** ***************************************************************************/ - public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List headerRow) + public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List headerRow, boolean isBulkEdit) { massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>(); for(int i = 0; i < headerRow.size(); i++) @@ -90,6 +90,7 @@ public class BulkLoadMappingSuggester .withVersion("v1") .withLayout(layout) .withHasHeaderRow(true) + .withIsBulkEdit(isBulkEdit) .withFieldList(fieldList); return (bulkLoadProfile); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java index 26575be3..77114b1c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java @@ -25,12 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.bulk.TableKeyFieldsPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; @@ -44,12 +51,16 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class BulkLoadTableStructureBuilder { + private static final QLogger LOG = QLogger.getLogger(BulkLoadTableStructureBuilder.class); + + + /*************************************************************************** ** ***************************************************************************/ public static BulkLoadTableStructure buildTableStructure(String tableName) { - return (buildTableStructure(tableName, null, null)); + return (buildTableStructure(tableName, null, null, false)); } @@ -57,13 +68,24 @@ public class BulkLoadTableStructureBuilder /*************************************************************************** ** ***************************************************************************/ - private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) + public static BulkLoadTableStructure buildTableStructure(String tableName, Boolean isBulkEdit) + { + return (buildTableStructure(tableName, null, null, isBulkEdit)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath, Boolean isBulkEdit) { QTableMetaData table = QContext.getQInstance().getTable(tableName); BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); tableStructure.setTableName(tableName); tableStructure.setLabel(table.getLabel()); + tableStructure.setIsBulkEdit(isBulkEdit); Set associationJoinFieldNamesToExclude = new HashSet<>(); @@ -119,6 +141,30 @@ public class BulkLoadTableStructureBuilder } } + //////////////////////////////////////////////////////// + // for bulk edit, users can use the primary key field // + //////////////////////////////////////////////////////// + if(isBulkEdit) + { + fields.add(table.getField(table.getPrimaryKeyField())); + + ////////////////////////////////////////////////////////////////////// + // also make available what key fields are available for this table // + ////////////////////////////////////////////////////////////////////// + try + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput() + .withPossibleValueSourceName("tableKeyFields") + .withPathParamMap(Map.of("processName", tableName + ".bulkEditWithFile")); + List> search = new TableKeyFieldsPossibleValueSource().search(input); + tableStructure.setPossibleKeyFields(new ArrayList<>(search.stream().map(QPossibleValue::getId).toList())); + } + catch(QException qe) + { + LOG.warn("Unable to retrieve possible key fields for table [" + tableName + "]", qe); + } + } + fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), ""))); for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) @@ -131,8 +177,8 @@ public class BulkLoadTableStructureBuilder { String nextLevelPath = (StringUtils.hasContent(parentAssociationPath) ? parentAssociationPath + "." : "") - + (association != null ? association.getName() : ""); - BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, nextLevelPath); + + (association != null ? association.getName() : ""); + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, nextLevelPath, isBulkEdit); tableStructure.addAssociation(associatedStructure); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java index 2fe07b3c..493615d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java @@ -37,6 +37,8 @@ public class BulkLoadProfile implements Serializable private Boolean hasHeaderRow; private String layout; private String version; + private Boolean isBulkEdit; + private String keyFields; @@ -132,6 +134,7 @@ public class BulkLoadProfile implements Serializable } + /******************************************************************************* ** Getter for version *******************************************************************************/ @@ -162,4 +165,65 @@ public class BulkLoadProfile implements Serializable } + + /******************************************************************************* + ** Getter for isBulkEdit + *******************************************************************************/ + public Boolean getIsBulkEdit() + { + return (this.isBulkEdit); + } + + + + /******************************************************************************* + ** Setter for isBulkEdit + *******************************************************************************/ + public void setIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + } + + + + /******************************************************************************* + ** Fluent setter for isBulkEdit + *******************************************************************************/ + public BulkLoadProfile withIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + return (this); + } + + + + /******************************************************************************* + ** Getter for keyFields + *******************************************************************************/ + public String getKeyFields() + { + return (this.keyFields); + } + + + + /******************************************************************************* + ** Setter for keyFields + *******************************************************************************/ + public void setKeyFields(String keyFields) + { + this.keyFields = keyFields; + } + + + + /******************************************************************************* + ** Fluent setter for keyFields + *******************************************************************************/ + public BulkLoadProfile withKeyFields(String keyFields) + { + this.keyFields = keyFields; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java index f7ce0f6c..17acc4c1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java @@ -35,6 +35,7 @@ public class BulkLoadProfileField private Integer columnIndex; private String headerName; private Serializable defaultValue; + private Boolean clearIfEmpty; private Boolean doValueMapping; private Map valueMappings; @@ -194,6 +195,7 @@ public class BulkLoadProfileField } + /******************************************************************************* ** Getter for headerName *******************************************************************************/ @@ -224,4 +226,34 @@ public class BulkLoadProfileField } + + /******************************************************************************* + ** Getter for clearIfEmpty + *******************************************************************************/ + public Boolean getClearIfEmpty() + { + return (this.clearIfEmpty); + } + + + + /******************************************************************************* + ** Setter for clearIfEmpty + *******************************************************************************/ + public void setClearIfEmpty(Boolean clearIfEmpty) + { + this.clearIfEmpty = clearIfEmpty; + } + + + + /******************************************************************************* + ** Fluent setter for clearIfEmpty + *******************************************************************************/ + public BulkLoadProfileField withClearIfEmpty(Boolean clearIfEmpty) + { + this.clearIfEmpty = clearIfEmpty; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java index db55198f..ff4e729b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java @@ -35,9 +35,12 @@ public class BulkLoadTableStructure implements Serializable private boolean isMain; private boolean isMany; - private String tableName; - private String label; - private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild + private String tableName; + private String label; + private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild + private Boolean isBulkEdit; + private String keyFields; + private ArrayList possibleKeyFields; private ArrayList fields; // mmm, not marked as serializable (at this time) - is okay? private ArrayList associations; @@ -272,4 +275,98 @@ public class BulkLoadTableStructure implements Serializable } this.associations.add(association); } + + + + /******************************************************************************* + ** Getter for isBulkEdit + *******************************************************************************/ + public Boolean getIsBulkEdit() + { + return (this.isBulkEdit); + } + + + + /******************************************************************************* + ** Setter for isBulkEdit + *******************************************************************************/ + public void setIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + } + + + + /******************************************************************************* + ** Fluent setter for isBulkEdit + *******************************************************************************/ + public BulkLoadTableStructure withIsBulkEdit(Boolean isBulkEdit) + { + this.isBulkEdit = isBulkEdit; + return (this); + } + + + + /******************************************************************************* + ** Getter for keyFields + *******************************************************************************/ + public String getKeyFields() + { + return (this.keyFields); + } + + + + /******************************************************************************* + ** Setter for keyFields + *******************************************************************************/ + public void setKeyFields(String keyFields) + { + this.keyFields = keyFields; + } + + + + /******************************************************************************* + ** Fluent setter for keyFields + *******************************************************************************/ + public BulkLoadTableStructure withKeyFields(String keyFields) + { + this.keyFields = keyFields; + return (this); + } + + + + /******************************************************************************* + ** Getter for possibleKeyFields + *******************************************************************************/ + public ArrayList getPossibleKeyFields() + { + return (this.possibleKeyFields); + } + + + + /******************************************************************************* + ** Setter for possibleKeyFields + *******************************************************************************/ + public void setPossibleKeyFields(ArrayList possibleKeyFields) + { + this.possibleKeyFields = possibleKeyFields; + } + + + + /******************************************************************************* + ** Fluent setter for possibleKeyFields + *******************************************************************************/ + public BulkLoadTableStructure withPossibleKeyFields(ArrayList possibleKeyFields) + { + this.possibleKeyFields = possibleKeyFields; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java index c1bdaa41..a6e6c3b5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java @@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import org.apache.commons.lang.BooleanUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -102,13 +103,29 @@ public class QuerySavedBulkLoadProfileProcess implements BackendStep } else { - String tableName = runBackendStepInput.getValueString("tableName"); + String tableName = runBackendStepInput.getValueString("tableName"); + boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit")); QueryInput input = new QueryInput(); input.setTableName(SavedBulkLoadProfile.TABLE_NAME); - input.setFilter(new QQueryFilter() + + QQueryFilter filter = new QQueryFilter() .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)) - .withOrderBy(new QFilterOrderBy("label"))); + .withOrderBy(new QFilterOrderBy("label")); + + ///////////////////////////////////////////////////////////////////// + // account for nulls here, so if is bulk edit, only look for true, // + // otherwise look for nulls or not equal to true // + ///////////////////////////////////////////////////////////////////// + if(isBulkEdit) + { + filter.withCriteria(new QFilterCriteria("isBulkEdit", QCriteriaOperator.EQUALS, true)); + } + else + { + filter.withCriteria(new QFilterCriteria("isBulkEdit", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, true)); + } + input.setFilter(filter); QueryOutput output = new QueryAction().execute(input); runBackendStepOutput.setRecords(output.getRecords()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java index 0d8f33d4..5a2e3eab 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java @@ -50,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.commons.lang.BooleanUtils; /******************************************************************************* @@ -87,9 +88,10 @@ public class StoreSavedBulkLoadProfileProcess implements BackendStep try { - String userId = QContext.getQSession().getUser().getIdReference(); - String tableName = runBackendStepInput.getValueString("tableName"); - String label = runBackendStepInput.getValueString("label"); + String userId = QContext.getQSession().getUser().getIdReference(); + String tableName = runBackendStepInput.getValueString("tableName"); + String label = runBackendStepInput.getValueString("label"); + Boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit")); String mappingJson = processMappingJson(runBackendStepInput.getValueString("mappingJson")); @@ -98,6 +100,7 @@ public class StoreSavedBulkLoadProfileProcess implements BackendStep .withValue("mappingJson", mappingJson) .withValue("label", label) .withValue("tableName", tableName) + .withValue("isBulkEdit", isBulkEdit) .withValue("userId", userId); List savedBulkLoadProfileList; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java index a54e08ef..21ad3a09 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java @@ -38,7 +38,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* - ** Unit test for BulkLoadMappingSuggester + ** Unit test for BulkLoadMappingSuggester *******************************************************************************/ class BulkLoadMappingSuggesterTest extends BaseTest { @@ -52,7 +52,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY); List headerRow = List.of("Id", "First Name", "lastname", "email", "homestate"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("FLAT", bulkLoadProfile.getLayout()); assertNull(getFieldByName(bulkLoadProfile, "id")); @@ -73,7 +73,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "shipto name", "sku", "quantity"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("TALL", bulkLoadProfile.getLayout()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); @@ -93,7 +93,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("TALL", bulkLoadProfile.getLayout()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); @@ -120,7 +120,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); @@ -136,7 +136,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex()); @@ -152,7 +152,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "ship to name", "address", "address 2"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); @@ -177,7 +177,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); List headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2"); - BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false); assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("WIDE", bulkLoadProfile.getLayout()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); @@ -206,4 +206,4 @@ class BulkLoadMappingSuggesterTest extends BaseTest .findFirst().orElse(null)); } -} \ No newline at end of file +} diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index a127b9f7..96de99d3 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -1932,6 +1932,8 @@ public class QJavalinImplementation input.setPossibleValueSourceName(possibleValueSourceName); input.setSearchTerm(searchTerm); input.setDefaultQueryFilter(defaultFilter); + input.setPathParamMap(context.pathParamMap()); + input.setQueryParamMap(context.queryParamMap()); if(StringUtils.hasContent(ids)) {