QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

initial checkin of support of bulk load with file

Browse Source
This commit is contained in:
Tim Chamberlain
2025-07-17 13:21:49 -05:00
parent e1eb07697a
commit 0a236b8c36
20 changed files with 1073 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java
View File

@ -57,7 +57,7 @@ public class BulkTableActionProcessPermissionChecker implements CustomPermission
switch(bulkActionName) switch(bulkActionName)
{ {
case "bulkInsert" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.INSERT); case "bulkInsert" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.INSERT);
case "bulkEdit" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.EDIT); case "bulkEdit", "bulkEditWithFile" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.EDIT);
case "bulkDelete" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.DELETE); case "bulkDelete" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.DELETE);
default -> LOG.warn("Unexpected bulk action name when checking permissions for process: " + processName); default -> LOG.warn("Unexpected bulk action name when checking permissions for process: " + processName);
} }

143
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvide
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface;
import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.bulk.TableKeyFieldsPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
@ -857,6 +858,8 @@ public class QInstanceEnricher
*******************************************************************************/ *******************************************************************************/
private void defineTableBulkProcesses(QInstance qInstance) private void defineTableBulkProcesses(QInstance qInstance)
{ {
qInstance.addPossibleValueSource(defineTableKeyFieldsPossibleValueSource());
for(QTableMetaData table : qInstance.getTables().values()) for(QTableMetaData table : qInstance.getTables().values())
{ {
if(table.getFields() == null) if(table.getFields() == null)
@ -880,6 +883,12 @@ public class QInstanceEnricher
defineTableBulkEdit(qInstance, table, bulkEditProcessName); defineTableBulkEdit(qInstance, table, bulkEditProcessName);
} }
String bulkEditWithFileProcessName = table.getName() + ".bulkEditWithFile";
if(qInstance.getProcess(bulkEditWithFileProcessName) == null)
{
defineTableBulkEditWithFile(qInstance, table, bulkEditWithFileProcessName);
}
String bulkDeleteProcessName = table.getName() + ".bulkDelete"; String bulkDeleteProcessName = table.getName() + ".bulkDelete";
if(qInstance.getProcess(bulkDeleteProcessName) == null) if(qInstance.getProcess(bulkDeleteProcessName) == null)
{ {
@ -984,16 +993,16 @@ public class QInstanceEnricher
.withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class)); .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class));
int i = 0; int i = 0;
process.addStep(i++, prepareFileUploadStep); process.withStep(i++, prepareFileUploadStep);
process.addStep(i++, uploadScreen); process.withStep(i++, uploadScreen);
process.addStep(i++, prepareFileMappingStep); process.withStep(i++, prepareFileMappingStep);
process.addStep(i++, fileMappingScreen); process.withStep(i++, fileMappingScreen);
process.addStep(i++, receiveFileMappingStep); process.withStep(i++, receiveFileMappingStep);
process.addStep(i++, prepareValueMappingStep); process.withStep(i++, prepareValueMappingStep);
process.addStep(i++, valueMappingScreen); process.withStep(i++, valueMappingScreen);
process.addStep(i++, receiveValueMappingStep); process.withStep(i++, receiveValueMappingStep);
process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields); process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields);
@ -1023,7 +1032,7 @@ public class QInstanceEnricher
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE); values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE);
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
ExtractViaQueryStep.class, BulkInsertExtractStep.class,
BulkEditTransformStep.class, BulkEditTransformStep.class,
BulkEditLoadStep.class, BulkEditLoadStep.class,
values values
@ -1060,6 +1069,122 @@ public class QInstanceEnricher
/*******************************************************************************
**
*******************************************************************************/
public void defineTableBulkEditWithFile(QInstance qInstance, QTableMetaData table, String processName)
{
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "This is a preview of the records that will be updated.");
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
BulkInsertExtractStep.class,
BulkInsertTransformStep.class,
BulkEditLoadStep.class,
values
)
.withName(processName)
.withLabel(table.getLabel() + " Bulk Edit With File")
.withTableName(table.getName())
.withIsHidden(true)
.withPermissionRules(qInstance.getDefaultPermissionRules().clone()
.withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class)));
List<QFieldMetaData> editableFields = table.getFields().values().stream()
.filter(QFieldMetaData::getIsEditable)
.filter(f -> !f.getType().equals(QFieldType.BLOB))
.toList();
QBackendStepMetaData prepareFileUploadStep = new QBackendStepMetaData()
.withName("prepareFileUpload")
.withCode(new QCodeReference(BulkInsertPrepareFileUploadStep.class));
QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData()
.withName("upload")
.withLabel("Upload File")
.withFormField(new QFieldMetaData("theFile", QFieldType.BLOB)
.withFieldAdornment(FileUploadAdornment.newFieldAdornment()
.withValue(FileUploadAdornment.formatDragAndDrop())
.withValue(FileUploadAdornment.widthFull()))
.withLabel(table.getLabel() + " File")
.withIsRequired(true))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM));
QBackendStepMetaData prepareFileMappingStep = new QBackendStepMetaData()
.withName("prepareFileMapping")
.withCode(new QCodeReference(BulkInsertPrepareFileMappingStep.class));
QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData()
.withName("fileMapping")
.withLabel("File Mapping")
.withBackStepName("prepareFileUpload")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM))
.withFormField(new QFieldMetaData("hasHeaderRow", QFieldType.BOOLEAN))
.withFormField(new QFieldMetaData("layout", QFieldType.STRING)) // is actually PVS, but, this field is only added to help support helpContent, so :shrug:
.withFormField(new QFieldMetaData("tableKeyFields", QFieldType.STRING).withPossibleValueSourceName(TableKeyFieldsPossibleValueSource.NAME));
QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData()
.withName("receiveFileMapping")
.withCode(new QCodeReference(BulkInsertReceiveFileMappingStep.class));
QBackendStepMetaData prepareValueMappingStep = new QBackendStepMetaData()
.withName("prepareValueMapping")
.withCode(new QCodeReference(BulkInsertPrepareValueMappingStep.class));
QFrontendStepMetaData valueMappingScreen = new QFrontendStepMetaData()
.withName("valueMapping")
.withLabel("Value Mapping")
.withBackStepName("prepareFileMapping")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM));
QBackendStepMetaData receiveValueMappingStep = new QBackendStepMetaData()
.withName("receiveValueMapping")
.withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class));
int i = 0;
process.withStep(i++, prepareFileUploadStep);
process.withStep(i++, uploadScreen);
process.withStep(i++, prepareFileMappingStep);
process.withStep(i++, fileMappingScreen);
process.withStep(i++, receiveFileMappingStep);
process.withStep(i++, prepareValueMappingStep);
process.withStep(i++, valueMappingScreen);
process.withStep(i++, receiveValueMappingStep);
process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields);
//////////////////////////////////////////////////////////////////////////////////////////
// put the bulk-load profile form (e.g., for saving it) on the review & result screens) //
//////////////////////////////////////////////////////////////////////////////////////////
process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)
.withBackStepName("prepareFileMapping")
.getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM));
process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_RESULT)
.getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM));
qInstance.addProcess(process);
}
/*******************************************************************************
**
*******************************************************************************/
private QPossibleValueSource defineTableKeyFieldsPossibleValueSource()
{
return (new QPossibleValueSource()
.withName(TableKeyFieldsPossibleValueSource.NAME)
.withType(QPossibleValueSourceType.CUSTOM)
.withCustomCodeReference(new QCodeReference(TableKeyFieldsPossibleValueSource.class)));
}
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/

76
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.values;
import java.io.Serializable; import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@ -34,11 +35,13 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
*******************************************************************************/ *******************************************************************************/
public class SearchPossibleValueSourceInput extends AbstractActionInput implements Cloneable public class SearchPossibleValueSourceInput extends AbstractActionInput implements Cloneable
{ {
private String possibleValueSourceName; private String possibleValueSourceName;
private QQueryFilter defaultQueryFilter; private QQueryFilter defaultQueryFilter;
private String searchTerm; private String searchTerm;
private List<Serializable> idList; private List<Serializable> idList;
private List<String> labelList; private List<String> labelList;
private Map<String, String> pathParamMap;
private Map<String, List<String>> queryParamMap;
private Integer skip = 0; private Integer skip = 0;
private Integer limit = 250; private Integer limit = 250;
@ -284,6 +287,7 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen
} }
/******************************************************************************* /*******************************************************************************
** Getter for labelList ** Getter for labelList
*******************************************************************************/ *******************************************************************************/
@ -313,4 +317,66 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen
return (this); return (this);
} }
/*******************************************************************************
** Getter for pathParamMap
*******************************************************************************/
public Map<String, String> getPathParamMap()
{
return (this.pathParamMap);
}
/*******************************************************************************
** Setter for pathParamMap
*******************************************************************************/
public void setPathParamMap(Map<String, String> pathParamMap)
{
this.pathParamMap = pathParamMap;
}
/*******************************************************************************
** Fluent setter for pathParamMap
*******************************************************************************/
public SearchPossibleValueSourceInput withPathParamMap(Map<String, String> pathParamMap)
{
this.pathParamMap = pathParamMap;
return (this);
}
/*******************************************************************************
** Getter for queryParamMap
*******************************************************************************/
public Map<String, List<String>> getQueryParamMap()
{
return (this.queryParamMap);
}
/*******************************************************************************
** Setter for queryParamMap
*******************************************************************************/
public void setQueryParamMap(Map<String, List<String>> queryParamMap)
{
this.queryParamMap = queryParamMap;
}
/*******************************************************************************
** Fluent setter for queryParamMap
*******************************************************************************/
public SearchPossibleValueSourceInput withQueryParamMap(Map<String, List<String>> queryParamMap)
{
this.queryParamMap = queryParamMap;
return (this);
}
} }

153
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/bulk/TableKeyFieldsPossibleValueSource.java Normal file
View File

@ -0,0 +1,153 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.bulk;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
**
*******************************************************************************/
public class TableKeyFieldsPossibleValueSource implements QCustomPossibleValueProvider<String>
{
public static final String NAME = "tableKeyFields";
/*******************************************************************************
**
*******************************************************************************/
@Override
public QPossibleValue<String> getPossibleValue(Serializable tableAndKey)
{
QPossibleValue<String> possibleValue = null;
/////////////////////////////////////////////////////////////
// keys are in the format <tableName>-<key1>|<key2>|<key3> //
/////////////////////////////////////////////////////////////
String[] keyParts = tableAndKey.toString().split("-");
String tableName = keyParts[0];
String key = keyParts[1];
QTableMetaData table = QContext.getQInstance().getTable(tableName);
if(table.getPrimaryKeyField().equals(key))
{
String id = table.getPrimaryKeyField();
String label = table.getField(table.getPrimaryKeyField()).getLabel();
possibleValue = new QPossibleValue<>(id, label);
}
else
{
for(UniqueKey uniqueKey : table.getUniqueKeys())
{
String potentialMatch = getIdFromUniqueKey(uniqueKey);
if(potentialMatch.equals(key))
{
String id = potentialMatch;
String label = getLabelFromUniqueKey(table, uniqueKey);
possibleValue = new QPossibleValue<>(id, label);
break;
}
}
}
return (possibleValue);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
{
List<QPossibleValue<String>> rs = new ArrayList<>();
if(!CollectionUtils.nonNullMap(input.getPathParamMap()).containsKey("processName") || input.getPathParamMap().get("processName") == null || input.getPathParamMap().get("processName").isEmpty())
{
throw (new QException("Path Param of processName was not found."));
}
////////////////////////////////////////////////////
// process name will be like tnt.bulkEditWithFile //
////////////////////////////////////////////////////
String processName = input.getPathParamMap().get("processName");
String tableName = processName.split("\\.")[0];
QTableMetaData table = QContext.getQInstance().getTable(tableName);
for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys()))
{
String id = getIdFromUniqueKey(uniqueKey);
String label = getLabelFromUniqueKey(table, uniqueKey);
if(!StringUtils.hasContent(input.getSearchTerm()) || input.getSearchTerm().equals(id))
{
rs.add(new QPossibleValue<>(id, label));
}
}
rs.sort(Comparator.comparing(QPossibleValue::getLabel));
///////////////////////////////
// put the primary key first //
///////////////////////////////
if(!StringUtils.hasContent(input.getSearchTerm()) || input.getSearchTerm().equals(table.getPrimaryKeyField()))
{
rs.add(0, new QPossibleValue<>(table.getPrimaryKeyField(), table.getField(table.getPrimaryKeyField()).getLabel()));
}
return rs;
}
/*******************************************************************************
**
*******************************************************************************/
private String getIdFromUniqueKey(UniqueKey uniqueKey)
{
return (StringUtils.join("|", uniqueKey.getFieldNames()));
}
/*******************************************************************************
**
*******************************************************************************/
private String getLabelFromUniqueKey(QTableMetaData tableMetaData, UniqueKey uniqueKey)
{
List<String> fieldLabels = new ArrayList<>(uniqueKey.getFieldNames().stream().map(f -> tableMetaData.getField(f).getLabel()).toList());
fieldLabels.sort(Comparator.naturalOrder());
return (StringUtils.joinWithCommasAndAnd(fieldLabels));
}
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java
View File

@ -60,6 +60,9 @@ public class SavedBulkLoadProfile extends QRecordEntity
@QField(label = "Mapping JSON") @QField(label = "Mapping JSON")
private String mappingJson; private String mappingJson;
@QField()
private Boolean isBulkEdit;
/******************************************************************************* /*******************************************************************************
@ -251,7 +254,6 @@ public class SavedBulkLoadProfile extends QRecordEntity
/******************************************************************************* /*******************************************************************************
** Getter for mappingJson ** Getter for mappingJson
*******************************************************************************/ *******************************************************************************/
@ -282,4 +284,34 @@ public class SavedBulkLoadProfile extends QRecordEntity
} }
/*******************************************************************************
** Getter for isBulkEdit
*******************************************************************************/
public Boolean getIsBulkEdit()
{
return (this.isBulkEdit);
}
/*******************************************************************************
** Setter for isBulkEdit
*******************************************************************************/
public void setIsBulkEdit(Boolean isBulkEdit)
{
this.isBulkEdit = isBulkEdit;
}
/*******************************************************************************
** Fluent setter for isBulkEdit
*******************************************************************************/
public SavedBulkLoadProfile withIsBulkEdit(Boolean isBulkEdit)
{
this.isBulkEdit = isBulkEdit;
return (this);
}
} }

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java
View File

@ -113,7 +113,7 @@ public class SavedBulkLoadProfileMetaDataProvider
.withFieldsFromEntity(SavedBulkLoadProfile.class) .withFieldsFromEntity(SavedBulkLoadProfile.class)
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName")))
.withSection(new QFieldSection("details", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "mappingJson"))) .withSection(new QFieldSection("details", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "mappingJson", "isBulkEdit")))
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
table.getField("mappingJson").withBehavior(SavedBulkLoadProfileJsonFieldDisplayValueFormatter.getInstance()); table.getField("mappingJson").withBehavior(SavedBulkLoadProfileJsonFieldDisplayValueFormatter.getInstance());

94
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java
View File

@ -36,10 +36,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.commons.lang.BooleanUtils;
import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.buildInfoSummaryLines; import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.buildInfoSummaryLines;
@ -53,6 +55,9 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private List<ProcessSummaryLine> infoSummaries = new ArrayList<>(); private List<ProcessSummaryLine> infoSummaries = new ArrayList<>();
private Serializable firstInsertedPrimaryKey = null;
private Serializable lastInsertedPrimaryKey = null;
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited"); private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited");
private String tableLabel; private String tableLabel;
@ -106,7 +111,15 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar
tableLabel = table.getLabel(); tableLabel = table.getLabel();
} }
buildInfoSummaryLines(runBackendStepInput, table, infoSummaries, true); boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit"));
if(isBulkEdit)
{
buildBulkUpdateWithFileInfoSummaryLines(runBackendStepOutput, table);
}
else
{
buildInfoSummaryLines(runBackendStepInput, table, infoSummaries, true);
}
} }
@ -146,4 +159,83 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar
} }
} }
/***************************************************************************
**
***************************************************************************/
private void buildBulkUpdateWithFileInfoSummaryLines(RunBackendStepOutput runBackendStepOutput, QTableMetaData table)
{
/////////////////////////////////////////////////////////////////////////////////////////////
// the transform step builds summary lines that it predicts will update successfully. //
// but those lines don't have ids, which we'd like to have (e.g., for a process trace that //
// might link to the built record). also, it's possible that there was a fail that only //
// happened in the actual update, so, basically, re-do the summary here //
/////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep();
ProcessSummaryLine okSummary = transformStep.okSummary;
okSummary.setCount(0);
okSummary.setPrimaryKeys(new ArrayList<>());
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// but - since errors from the transform step don't even make it through to us here in the load step, //
// do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows: //
// clear out its warnings - we'll completely rebuild them here (with primary keys) //
// and add new error lines, e.g., in case of errors that only happened past the validation if possible. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup;
processSummaryWarningsAndErrorsRollup.resetWarnings();
List<QRecord> updatedRecords = runBackendStepOutput.getRecords();
for(QRecord updatedRecord : updatedRecords)
{
Serializable primaryKey = updatedRecord.getValue(table.getPrimaryKeyField());
if(CollectionUtils.nullSafeIsEmpty(updatedRecord.getErrors()) && primaryKey != null)
{
/////////////////////////////////////////////////////////////////////////
// if the record had no errors, and we have a primary key for it, then //
// keep track of the range of primary keys (first and last) //
/////////////////////////////////////////////////////////////////////////
if(firstInsertedPrimaryKey == null)
{
firstInsertedPrimaryKey = primaryKey;
}
lastInsertedPrimaryKey = primaryKey;
if(!CollectionUtils.nullSafeIsEmpty(updatedRecord.getWarnings()))
{
////////////////////////////////////////////////////////////////////////////
// if there were warnings on the updated record, put it in a warning line //
////////////////////////////////////////////////////////////////////////////
String message = updatedRecord.getWarnings().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey);
}
else
{
///////////////////////////////////////////////////////////////////////
// if no warnings for the updated record, then put it in the OK line //
///////////////////////////////////////////////////////////////////////
okSummary.incrementCountAndAddPrimaryKey(primaryKey);
}
}
else
{
//////////////////////////////////////////////////////////////////////
// else if there were errors or no primary key, build an error line //
//////////////////////////////////////////////////////////////////////
String message = "Failed to update";
if(!CollectionUtils.nullSafeIsEmpty(updatedRecord.getErrors()))
{
//////////////////////////////////////////////////////////
// use the error message from the record if we have one //
//////////////////////////////////////////////////////////
message = updatedRecord.getErrors().get(0).getMessage();
}
processSummaryWarningsAndErrorsRollup.addError(message, primaryKey);
}
}
okSummary.pickMessage(true);
}
} }

17
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java
View File

@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang.BooleanUtils;
import org.json.JSONObject; import org.json.JSONObject;
@ -65,9 +66,11 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
{ {
buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput);
boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit"));
String tableName = runBackendStepInput.getValueString("tableName"); String tableName = runBackendStepInput.getValueString("tableName");
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName, isBulkEdit);
runBackendStepOutput.addValue("tableStructure", tableStructure); runBackendStepOutput.addValue("tableStructure", tableStructure);
runBackendStepOutput.addValue("isBulkEdit", isBulkEdit);
boolean needSuggestedMapping = true; boolean needSuggestedMapping = true;
if(runBackendStepOutput.getProcessState().getIsStepBack()) if(runBackendStepOutput.getProcessState().getIsStepBack())
@ -81,7 +84,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
{ {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
List<String> headerValues = (List<String>) runBackendStepOutput.getValue("headerValues"); List<String> headerValues = (List<String>) runBackendStepOutput.getValue("headerValues");
buildSuggestedMapping(headerValues, getPrepopulatedValues(runBackendStepInput), tableStructure, runBackendStepOutput); buildSuggestedMapping(isBulkEdit, headerValues, getPrepopulatedValues(runBackendStepInput), tableStructure, runBackendStepOutput);
} }
} }
@ -95,8 +98,8 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
String prepopulatedValuesJson = runBackendStepInput.getValueString("prepopulatedValues"); String prepopulatedValuesJson = runBackendStepInput.getValueString("prepopulatedValues");
if(StringUtils.hasContent(prepopulatedValuesJson)) if(StringUtils.hasContent(prepopulatedValuesJson))
{ {
Map<String, Serializable> rs = new LinkedHashMap<>(); Map<String, Serializable> rs = new LinkedHashMap<>();
JSONObject jsonObject = JsonUtils.toJSONObject(prepopulatedValuesJson); JSONObject jsonObject = JsonUtils.toJSONObject(prepopulatedValuesJson);
for(String key : jsonObject.keySet()) for(String key : jsonObject.keySet())
{ {
rs.put(key, jsonObject.optString(key, null)); rs.put(key, jsonObject.optString(key, null));
@ -112,16 +115,16 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
/*************************************************************************** /***************************************************************************
** **
***************************************************************************/ ***************************************************************************/
private void buildSuggestedMapping(List<String> headerValues, Map<String, Serializable> prepopulatedValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) private void buildSuggestedMapping(boolean isBulkEdit, List<String> headerValues, Map<String, Serializable> prepopulatedValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput)
{ {
BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester();
BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues, isBulkEdit);
if(CollectionUtils.nullSafeHasContents(prepopulatedValues)) if(CollectionUtils.nullSafeHasContents(prepopulatedValues))
{ {
for(Map.Entry<String, Serializable> entry : prepopulatedValues.entrySet()) for(Map.Entry<String, Serializable> entry : prepopulatedValues.entrySet())
{ {
String fieldName = entry.getKey(); String fieldName = entry.getKey();
boolean foundFieldInProfile = false; boolean foundFieldInProfile = false;
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java
View File

@ -65,10 +65,12 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
runBackendStepOutput.addValue("theFile", null); runBackendStepOutput.addValue("theFile", null);
} }
boolean isBulkEdit = runBackendStepInput.getProcessName().endsWith("EditWithFile");
String tableName = runBackendStepInput.getValueString("tableName"); String tableName = runBackendStepInput.getValueString("tableName");
QTableMetaData table = QContext.getQInstance().getTable(tableName); QTableMetaData table = QContext.getQInstance().getTable(tableName);
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName);
runBackendStepOutput.addValue("tableStructure", tableStructure); runBackendStepOutput.addValue("tableStructure", tableStructure);
runBackendStepOutput.addValue("isBulkEdit", isBulkEdit);
List<QFieldMetaData> requiredFields = new ArrayList<>(); List<QFieldMetaData> requiredFields = new ArrayList<>();
List<QFieldMetaData> additionalFields = new ArrayList<>(); List<QFieldMetaData> additionalFields = new ArrayList<>();
@ -84,6 +86,14 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
} }
} }
/////////////////////////////////////////////
// bulk edit allows primary key as a field //
/////////////////////////////////////////////
if(isBulkEdit)
{
requiredFields.add(0, table.getField(table.getPrimaryKeyField()));
}
StringBuilder html; StringBuilder html;
String childTableLabels = ""; String childTableLabels = "";
@ -96,11 +106,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////
boolean listFieldsInHelpText = false; boolean listFieldsInHelpText = false;
if(!CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) if(isBulkEdit || !CollectionUtils.nullSafeHasContents(tableStructure.getAssociations()))
{ {
html = new StringBuilder(""" html = new StringBuilder("""
<p>Upload either a CSV or Excel (.xlsx) file, with one row for each record you want to <p>Upload either a CSV or Excel (.xlsx) file, with one row for each record you want to
insert in the ${tableLabel} table.</p><br /> ${action} in the ${tableLabel} table.</p><br />
<p>Your file can contain any number of columns. You will be prompted to map fields from <p>Your file can contain any number of columns. You will be prompted to map fields from
the ${tableLabel} table to columns from your file or default values for all records that the ${tableLabel} table to columns from your file or default values for all records that
@ -204,6 +214,7 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
finishCSV(flatCSV); finishCSV(flatCSV);
String htmlString = html.toString() String htmlString = html.toString()
.replace("${action}", (isBulkEdit ? "edit" : "insert"))
.replace("${tableLabel}", table.getLabel()) .replace("${tableLabel}", table.getLabel())
.replace("${childTableLabels}", childTableLabels) .replace("${childTableLabels}", childTableLabels)
.replace("${flatCSV}", Base64.getEncoder().encodeToString(flatCSV.toString().getBytes(StandardCharsets.UTF_8))) .replace("${flatCSV}", Base64.getEncoder().encodeToString(flatCSV.toString().getBytes(StandardCharsets.UTF_8)))

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java
View File

@ -113,6 +113,8 @@ public class BulkInsertStepUtils
{ {
String layout = runBackendStepInput.getValueString("layout"); String layout = runBackendStepInput.getValueString("layout");
Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow"); Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow");
String keyFields = runBackendStepInput.getValueString("keyFields");
Boolean isBulkEdit = runBackendStepInput.getValueBoolean("isBulkEdit");
ArrayList<BulkLoadProfileField> fieldList = new ArrayList<>(); ArrayList<BulkLoadProfileField> fieldList = new ArrayList<>();
@ -127,6 +129,7 @@ public class BulkInsertStepUtils
bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null);
bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue"));
bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping"));
bulkLoadProfileField.setClearIfEmpty(jsonObject.optBoolean("clearIfEmpty"));
if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings")) if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings"))
{ {
@ -140,6 +143,8 @@ public class BulkInsertStepUtils
} }
BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() BulkLoadProfile bulkLoadProfile = new BulkLoadProfile()
.withIsBulkEdit(isBulkEdit)
.withKeyFields(keyFields)
.withVersion(version) .withVersion(version)
.withFieldList(fieldList) .withFieldList(fieldList)
.withHasHeaderRow(hasHeaderRow) .withHasHeaderRow(hasHeaderRow)
@ -213,7 +218,7 @@ public class BulkInsertStepUtils
{ {
return (processTracerKeyRecordMessage); return (processTracerKeyRecordMessage);
} }
return (null); return (null);
} }
} }

288
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable; import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
@ -32,12 +33,13 @@ import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -48,6 +50,11 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
@ -68,6 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.ListingHash;
import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang.BooleanUtils;
import org.json.JSONArray;
import org.json.JSONObject;
/******************************************************************************* /*******************************************************************************
@ -75,9 +85,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/ *******************************************************************************/
public class BulkInsertTransformStep extends AbstractTransformStep public class BulkInsertTransformStep extends AbstractTransformStep
{ {
ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); public ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") public ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
.withDoReplaceSingletonCountLinesWithSuffixOnly(false); .withDoReplaceSingletonCountLinesWithSuffixOnly(false);
private ListingHash<String, RowValue> errorToExampleRowValueMap = new ListingHash<>(); private ListingHash<String, RowValue> errorToExampleRowValueMap = new ListingHash<>();
@ -190,6 +200,252 @@ public class BulkInsertTransformStep extends AbstractTransformStep
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
List<QRecord> records = runBackendStepInput.getRecords(); List<QRecord> records = runBackendStepInput.getRecords();
if(BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit")))
{
handleBulkEdit(runBackendStepInput, runBackendStepOutput, records, table);
runBackendStepOutput.addValue("isBulkEdit", true);
}
else
{
handleBulkLoad(runBackendStepInput, runBackendStepOutput, records, table);
}
}
/*******************************************************************************
**
*******************************************************************************/
private void handleBulkEdit(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<QRecord> records, QTableMetaData table) throws QException
{
///////////////////////////////////////////
// get the key fields for this bulk edit //
///////////////////////////////////////////
String keyFieldsString = runBackendStepInput.getValueString("keyFields");
List<String> keyFields = Arrays.asList(keyFieldsString.split("\\|"));
//////////////////////////////////////////////////////////////////////////
// if the key field is the primary key, then just look up those records //
//////////////////////////////////////////////////////////////////////////
List<QRecord> nonMatchingRecords = new ArrayList<>();
List<QRecord> oldRecords = new ArrayList<>();
List<QRecord> recordsToUpdate = new ArrayList<>();
if(keyFields.size() == 1 && table.getPrimaryKeyField().equals(keyFields.get(0)))
{
recordsToUpdate = records;
String primaryKeyName = table.getPrimaryKeyField();
List<Serializable> primaryKeys = records.stream().map(record -> record.getValue(primaryKeyName)).toList();
oldRecords = new QueryAction().execute(new QueryInput(table.getName()).withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeys)))).getRecords();
///////////////////////////////////////////
// get a set of old records primary keys //
///////////////////////////////////////////
Set<Serializable> matchedPrimaryKeys = oldRecords.stream()
.map(r -> r.getValue(table.getPrimaryKeyField()))
.collect(java.util.stream.Collectors.toSet());
///////////////////////////////////////////////////////////////////////////////////////////////////
// iterate over file records and if primary keys dont match, add to the non matchin records list //
///////////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : records)
{
Serializable recordKey = record.getValue(table.getPrimaryKeyField());
if(!matchedPrimaryKeys.contains(recordKey))
{
nonMatchingRecords.add(record);
}
}
}
else
{
Set<Serializable> uniqueIds = new HashSet<>();
List<QRecord> potentialRecords = new ArrayList<>();
////////////////////////////////////////////////////////////////////////////////////////////////////
// if not using the primary key, then we will look up all records for each part of the unique key //
// and for each found, if all unique parts match we will add to our list of database records //
////////////////////////////////////////////////////////////////////////////////////////////////////
for(String uniqueKeyPart : keyFields)
{
List<Serializable> values = records.stream().map(record -> record.getValue(uniqueKeyPart)).toList();
for(QRecord databaseRecord : new QueryAction().execute(new QueryInput(table.getName()).withFilter(new QQueryFilter(new QFilterCriteria(uniqueKeyPart, QCriteriaOperator.IN, values)))).getRecords())
{
if(!uniqueIds.contains(databaseRecord.getValue(table.getPrimaryKeyField())))
{
potentialRecords.add(databaseRecord);
uniqueIds.add(databaseRecord.getValue(table.getPrimaryKeyField()));
}
}
}
///////////////////////////////////////////////////////////////////////////////
// now iterate over all of the potential records checking each unique fields //
///////////////////////////////////////////////////////////////////////////////
fileRecordLoop:
for(QRecord fileRecord : records)
{
for(QRecord databaseRecord : potentialRecords)
{
boolean allMatch = true;
for(String uniqueKeyPart : keyFields)
{
if(!Objects.equals(fileRecord.getValue(uniqueKeyPart), databaseRecord.getValue(uniqueKeyPart)))
{
allMatch = false;
}
}
//////////////////////////////////////////////////////////////////////////////////////
// if we get here with all matching, update the record from the file's primary key, //
// add it to the list to update, and continue looping over file records //
//////////////////////////////////////////////////////////////////////////////////////
if(allMatch)
{
oldRecords.add(databaseRecord);
fileRecord.setValue(table.getPrimaryKeyField(), databaseRecord.getValue(table.getPrimaryKeyField()));
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// iterate over the fields in the bulk load profile, if the value for that field is empty and the value //
// of 'clear if empty' is set to true, then update the record to update with the old record's value //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
JSONArray array = new JSONArray(runBackendStepInput.getValueString("fieldListJSON"));
for(int i = 0; i < array.length(); i++)
{
JSONObject jsonObject = array.getJSONObject(i);
String fieldName = jsonObject.optString("fieldName");
boolean clearIfEmpty = jsonObject.optBoolean("clearIfEmpty");
if(fileRecord.getValue(fieldName) == null)
{
if(clearIfEmpty)
{
fileRecord.setValue(fieldName, null);
}
else
{
fileRecord.setValue(fieldName, databaseRecord.getValue(fieldName));
}
}
}
recordsToUpdate.add(fileRecord);
continue fileRecordLoop;
}
}
///////////////////////////////////////////////////////////////////////////////////////
// if we make it here, that means the record was not found, keep for logging warning //
///////////////////////////////////////////////////////////////////////////////////////
nonMatchingRecords.add(fileRecord);
}
}
for(QRecord missingRecord : CollectionUtils.nonNullList(nonMatchingRecords))
{
String message = "Did not have a matching existing record.";
processSummaryWarningsAndErrorsRollup.addError(message, null);
addToErrorToExampleRowMap(message, missingRecord);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
UpdateInput updateInput = new UpdateInput(table.getName());
updateInput.setInputSource(QInputSource.USER);
updateInput.setRecords(recordsToUpdate);
//////////////////////////////////////////////////////////////////////
// load the pre-insert customizer and set it up, if there is one //
// then we'll run it based on its WhenToRun value //
// we do this, in case it needs to, for example, adjust values that //
// are part of a unique key //
//////////////////////////////////////////////////////////////////////
boolean didAlreadyRunCustomizer = false;
Optional<TableCustomizerInterface> preUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_UPDATE_RECORD.getRole());
if(preUpdateCustomizer.isPresent())
{
List<QRecord> recordsAfterCustomizer = preUpdateCustomizer.get().preUpdate(updateInput, records, true, Optional.of(oldRecords));
runBackendStepInput.setRecords(recordsAfterCustomizer);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// so we used to have a comment here asking "do we care if the customizer runs both now, and in the validation below?" //
// when implementing Bulk Load V2, we were seeing that some customizers were adding errors to records, both now, and //
// when they ran below. so, at that time, we added this boolean, to track and avoid the double-run... //
// we could also imagine this being a setting on the pre-insert customizer, similar to its whenToRun attribute... //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
didAlreadyRunCustomizer = true;
}
/////////////////////////////////////////////////////////////////////////////////
// run all validation from the insert action - in Preview mode (boolean param) //
/////////////////////////////////////////////////////////////////////////////////
updateInput.setRecords(recordsToUpdate);
UpdateAction updateAction = new UpdateAction();
updateAction.performValidations(updateInput, Optional.of(recordsToUpdate), didAlreadyRunCustomizer);
List<QRecord> validationResultRecords = updateInput.getRecords();
/////////////////////////////////////////////////////////////////
// look at validation results to build process summary results //
/////////////////////////////////////////////////////////////////
List<QRecord> outputRecords = new ArrayList<>();
for(QRecord record : validationResultRecords)
{
List<QErrorMessage> errorsFromAssociations = getErrorsFromAssociations(record);
if(CollectionUtils.nullSafeHasContents(errorsFromAssociations))
{
List<QErrorMessage> recordErrors = Objects.requireNonNullElseGet(record.getErrors(), () -> new ArrayList<>());
recordErrors.addAll(errorsFromAssociations);
record.setErrors(recordErrors);
}
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
for(QErrorMessage error : record.getErrors())
{
if(error instanceof AbstractBulkLoadRollableValueError rollableValueError)
{
processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null);
addToErrorToExampleRowValueMap(rollableValueError, record);
}
else
{
processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null);
addToErrorToExampleRowMap(error.getMessage(), record);
}
}
}
else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
{
String message = record.getWarnings().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addWarning(message, null);
outputRecords.add(record);
}
else
{
okSummary.incrementCountAndAddPrimaryKey(null);
outputRecords.add(record);
for(Map.Entry<String, List<QRecord>> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet())
{
String associationName = entry.getKey();
ProcessSummaryLine associationToInsertLine = associationsToInsertSummaries.computeIfAbsent(associationName, x -> new ProcessSummaryLine(Status.OK));
associationToInsertLine.incrementCount(CollectionUtils.nonNullList(entry.getValue()).size());
}
}
}
runBackendStepOutput.setRecords(outputRecords);
this.rowsProcessed += records.size();
}
/*******************************************************************************
**
*******************************************************************************/
private void handleBulkLoad(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<QRecord> records, QTableMetaData table) throws QException
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -209,7 +465,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
Optional<TableCustomizerInterface> preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); Optional<TableCustomizerInterface> preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole());
if(preInsertCustomizer.isPresent()) if(preInsertCustomizer.isPresent())
{ {
AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true);
if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun))
{ {
List<QRecord> recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, records, true); List<QRecord> recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, records, true);
@ -485,11 +741,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep
recordsProcessedLine.withPluralFutureMessage("records were"); recordsProcessedLine.withPluralFutureMessage("records were");
recordsProcessedLine.withPluralPastMessage("records were"); recordsProcessedLine.withPluralPastMessage("records were");
String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepOutput.getValueBoolean("isBulkEdit"));
okSummary.setSingularFutureMessage(tableLabel + " record will be inserted" + noWarningsSuffix + "."); String action = isBulkEdit ? "updated" : "inserted";
okSummary.setPluralFutureMessage(tableLabel + " records will be inserted" + noWarningsSuffix + "."); String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings";
okSummary.setSingularPastMessage(tableLabel + " record was inserted" + noWarningsSuffix + "."); okSummary.setSingularFutureMessage(tableLabel + " record will be " + action + noWarningsSuffix + ".");
okSummary.setPluralPastMessage(tableLabel + " records were inserted" + noWarningsSuffix + "."); okSummary.setPluralFutureMessage(tableLabel + " records will be " + action + noWarningsSuffix + ".");
okSummary.setSingularPastMessage(tableLabel + " record was " + action + noWarningsSuffix + ".");
okSummary.setPluralPastMessage(tableLabel + " records were " + action + noWarningsSuffix + ".");
okSummary.pickMessage(isForResultScreen); okSummary.pickMessage(isForResultScreen);
okSummary.addSelfToListIfAnyCount(rs); okSummary.addSelfToListIfAnyCount(rs);
@ -502,10 +760,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep
String associationLabel = associationTable.getLabel(); String associationLabel = associationTable.getLabel();
ProcessSummaryLine line = entry.getValue(); ProcessSummaryLine line = entry.getValue();
line.setSingularFutureMessage(associationLabel + " record will be inserted."); line.setSingularFutureMessage(associationLabel + " record will be " + action + ".");
line.setPluralFutureMessage(associationLabel + " records will be inserted."); line.setPluralFutureMessage(associationLabel + " records will be " + action + ".");
line.setSingularPastMessage(associationLabel + " record was inserted."); line.setSingularPastMessage(associationLabel + " record was " + action + ".");
line.setPluralPastMessage(associationLabel + " records were inserted."); line.setPluralPastMessage(associationLabel + " records were " + action + ".");
line.pickMessage(isForResultScreen); line.pickMessage(isForResultScreen);
line.addSelfToListIfAnyCount(rs); line.addSelfToListIfAnyCount(rs);
} }
@ -518,8 +776,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
ukErrorSummary ukErrorSummary
.withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values" .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values"
+ (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ")
+ StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues)))
.withSingularFutureMessage(" record will not be") .withSingularFutureMessage(" record will not be")
.withPluralFutureMessage(" records will not be") .withPluralFutureMessage(" records will not be")

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java
View File

@ -54,7 +54,7 @@ public class BulkLoadMappingSuggester
/*************************************************************************** /***************************************************************************
** **
***************************************************************************/ ***************************************************************************/
public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List<String> headerRow) public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List<String> headerRow, boolean isBulkEdit)
{ {
massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>(); massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>();
for(int i = 0; i < headerRow.size(); i++) for(int i = 0; i < headerRow.size(); i++)
@ -90,6 +90,7 @@ public class BulkLoadMappingSuggester
.withVersion("v1") .withVersion("v1")
.withLayout(layout) .withLayout(layout)
.withHasHeaderRow(true) .withHasHeaderRow(true)
.withIsBulkEdit(isBulkEdit)
.withFieldList(fieldList); .withFieldList(fieldList);
return (bulkLoadProfile); return (bulkLoadProfile);

54
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java
View File

@ -25,12 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set; import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.bulk.TableKeyFieldsPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
@ -44,12 +51,16 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
*******************************************************************************/ *******************************************************************************/
public class BulkLoadTableStructureBuilder public class BulkLoadTableStructureBuilder
{ {
private static final QLogger LOG = QLogger.getLogger(BulkLoadTableStructureBuilder.class);
/*************************************************************************** /***************************************************************************
** **
***************************************************************************/ ***************************************************************************/
public static BulkLoadTableStructure buildTableStructure(String tableName) public static BulkLoadTableStructure buildTableStructure(String tableName)
{ {
return (buildTableStructure(tableName, null, null)); return (buildTableStructure(tableName, null, null, false));
} }
@ -57,13 +68,24 @@ public class BulkLoadTableStructureBuilder
/*************************************************************************** /***************************************************************************
** **
***************************************************************************/ ***************************************************************************/
private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) public static BulkLoadTableStructure buildTableStructure(String tableName, Boolean isBulkEdit)
{
return (buildTableStructure(tableName, null, null, isBulkEdit));
}
/***************************************************************************
**
***************************************************************************/
private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath, Boolean isBulkEdit)
{ {
QTableMetaData table = QContext.getQInstance().getTable(tableName); QTableMetaData table = QContext.getQInstance().getTable(tableName);
BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); BulkLoadTableStructure tableStructure = new BulkLoadTableStructure();
tableStructure.setTableName(tableName); tableStructure.setTableName(tableName);
tableStructure.setLabel(table.getLabel()); tableStructure.setLabel(table.getLabel());
tableStructure.setIsBulkEdit(isBulkEdit);
Set<String> associationJoinFieldNamesToExclude = new HashSet<>(); Set<String> associationJoinFieldNamesToExclude = new HashSet<>();
@ -119,6 +141,30 @@ public class BulkLoadTableStructureBuilder
} }
} }
////////////////////////////////////////////////////////
// for bulk edit, users can use the primary key field //
////////////////////////////////////////////////////////
if(isBulkEdit)
{
fields.add(table.getField(table.getPrimaryKeyField()));
//////////////////////////////////////////////////////////////////////
// also make available what key fields are available for this table //
//////////////////////////////////////////////////////////////////////
try
{
SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput()
.withPossibleValueSourceName("tableKeyFields")
.withPathParamMap(Map.of("processName", tableName + ".bulkEditWithFile"));
List<QPossibleValue<String>> search = new TableKeyFieldsPossibleValueSource().search(input);
tableStructure.setPossibleKeyFields(new ArrayList<>(search.stream().map(QPossibleValue::getId).toList()));
}
catch(QException qe)
{
LOG.warn("Unable to retrieve possible key fields for table [" + tableName + "]", qe);
}
}
fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), ""))); fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), "")));
for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations()))
@ -131,8 +177,8 @@ public class BulkLoadTableStructureBuilder
{ {
String nextLevelPath = String nextLevelPath =
(StringUtils.hasContent(parentAssociationPath) ? parentAssociationPath + "." : "") (StringUtils.hasContent(parentAssociationPath) ? parentAssociationPath + "." : "")
+ (association != null ? association.getName() : ""); + (association != null ? association.getName() : "");
BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, nextLevelPath); BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, nextLevelPath, isBulkEdit);
tableStructure.addAssociation(associatedStructure); tableStructure.addAssociation(associatedStructure);
} }
} }

64
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java
View File

@ -37,6 +37,8 @@ public class BulkLoadProfile implements Serializable
private Boolean hasHeaderRow; private Boolean hasHeaderRow;
private String layout; private String layout;
private String version; private String version;
private Boolean isBulkEdit;
private String keyFields;
@ -132,6 +134,7 @@ public class BulkLoadProfile implements Serializable
} }
/******************************************************************************* /*******************************************************************************
** Getter for version ** Getter for version
*******************************************************************************/ *******************************************************************************/
@ -162,4 +165,65 @@ public class BulkLoadProfile implements Serializable
} }
/*******************************************************************************
** Getter for isBulkEdit
*******************************************************************************/
public Boolean getIsBulkEdit()
{
return (this.isBulkEdit);
}
/*******************************************************************************
** Setter for isBulkEdit
*******************************************************************************/
public void setIsBulkEdit(Boolean isBulkEdit)
{
this.isBulkEdit = isBulkEdit;
}
/*******************************************************************************
** Fluent setter for isBulkEdit
*******************************************************************************/
public BulkLoadProfile withIsBulkEdit(Boolean isBulkEdit)
{
this.isBulkEdit = isBulkEdit;
return (this);
}
/*******************************************************************************
** Getter for keyFields
*******************************************************************************/
public String getKeyFields()
{
return (this.keyFields);
}
/*******************************************************************************
** Setter for keyFields
*******************************************************************************/
public void setKeyFields(String keyFields)
{
this.keyFields = keyFields;
}
/*******************************************************************************
** Fluent setter for keyFields
*******************************************************************************/
public BulkLoadProfile withKeyFields(String keyFields)
{
this.keyFields = keyFields;
return (this);
}
} }

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java
View File

@ -35,6 +35,7 @@ public class BulkLoadProfileField
private Integer columnIndex; private Integer columnIndex;
private String headerName; private String headerName;
private Serializable defaultValue; private Serializable defaultValue;
private Boolean clearIfEmpty;
private Boolean doValueMapping; private Boolean doValueMapping;
private Map<String, Serializable> valueMappings; private Map<String, Serializable> valueMappings;
@ -194,6 +195,7 @@ public class BulkLoadProfileField
} }
/******************************************************************************* /*******************************************************************************
** Getter for headerName ** Getter for headerName
*******************************************************************************/ *******************************************************************************/
@ -224,4 +226,34 @@ public class BulkLoadProfileField
} }
/*******************************************************************************
** Getter for clearIfEmpty
*******************************************************************************/
public Boolean getClearIfEmpty()
{
return (this.clearIfEmpty);
}
/*******************************************************************************
** Setter for clearIfEmpty
*******************************************************************************/
public void setClearIfEmpty(Boolean clearIfEmpty)
{
this.clearIfEmpty = clearIfEmpty;
}
/*******************************************************************************
** Fluent setter for clearIfEmpty
*******************************************************************************/
public BulkLoadProfileField withClearIfEmpty(Boolean clearIfEmpty)
{
this.clearIfEmpty = clearIfEmpty;
return (this);
}
} }

103
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java
View File

@ -35,9 +35,12 @@ public class BulkLoadTableStructure implements Serializable
private boolean isMain; private boolean isMain;
private boolean isMany; private boolean isMany;
private String tableName; private String tableName;
private String label; private String label;
private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild
private Boolean isBulkEdit;
private String keyFields;
private ArrayList<String> possibleKeyFields;
private ArrayList<QFieldMetaData> fields; // mmm, not marked as serializable (at this time) - is okay? private ArrayList<QFieldMetaData> fields; // mmm, not marked as serializable (at this time) - is okay?
private ArrayList<BulkLoadTableStructure> associations; private ArrayList<BulkLoadTableStructure> associations;
@ -272,4 +275,98 @@ public class BulkLoadTableStructure implements Serializable
} }
this.associations.add(association); this.associations.add(association);
} }
/*******************************************************************************
** Getter for isBulkEdit
*******************************************************************************/
public Boolean getIsBulkEdit()
{
return (this.isBulkEdit);
}
/*******************************************************************************
** Setter for isBulkEdit
*******************************************************************************/
public void setIsBulkEdit(Boolean isBulkEdit)
{
this.isBulkEdit = isBulkEdit;
}
/*******************************************************************************
** Fluent setter for isBulkEdit
*******************************************************************************/
public BulkLoadTableStructure withIsBulkEdit(Boolean isBulkEdit)
{
this.isBulkEdit = isBulkEdit;
return (this);
}
/*******************************************************************************
** Getter for keyFields
*******************************************************************************/
public String getKeyFields()
{
return (this.keyFields);
}
/*******************************************************************************
** Setter for keyFields
*******************************************************************************/
public void setKeyFields(String keyFields)
{
this.keyFields = keyFields;
}
/*******************************************************************************
** Fluent setter for keyFields
*******************************************************************************/
public BulkLoadTableStructure withKeyFields(String keyFields)
{
this.keyFields = keyFields;
return (this);
}
/*******************************************************************************
** Getter for possibleKeyFields
*******************************************************************************/
public ArrayList<String> getPossibleKeyFields()
{
return (this.possibleKeyFields);
}
/*******************************************************************************
** Setter for possibleKeyFields
*******************************************************************************/
public void setPossibleKeyFields(ArrayList<String> possibleKeyFields)
{
this.possibleKeyFields = possibleKeyFields;
}
/*******************************************************************************
** Fluent setter for possibleKeyFields
*******************************************************************************/
public BulkLoadTableStructure withPossibleKeyFields(ArrayList<String> possibleKeyFields)
{
this.possibleKeyFields = possibleKeyFields;
return (this);
}
} }

23
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java
View File

@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
import org.apache.commons.lang.BooleanUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -102,13 +103,29 @@ public class QuerySavedBulkLoadProfileProcess implements BackendStep
} }
else else
{ {
String tableName = runBackendStepInput.getValueString("tableName"); String tableName = runBackendStepInput.getValueString("tableName");
boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit"));
QueryInput input = new QueryInput(); QueryInput input = new QueryInput();
input.setTableName(SavedBulkLoadProfile.TABLE_NAME); input.setTableName(SavedBulkLoadProfile.TABLE_NAME);
input.setFilter(new QQueryFilter()
QQueryFilter filter = new QQueryFilter()
.withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)) .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName))
.withOrderBy(new QFilterOrderBy("label"))); .withOrderBy(new QFilterOrderBy("label"));
/////////////////////////////////////////////////////////////////////
// account for nulls here, so if is bulk edit, only look for true, //
// otherwise look for nulls or not equal to true //
/////////////////////////////////////////////////////////////////////
if(isBulkEdit)
{
filter.withCriteria(new QFilterCriteria("isBulkEdit", QCriteriaOperator.EQUALS, true));
}
else
{
filter.withCriteria(new QFilterCriteria("isBulkEdit", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, true));
}
input.setFilter(filter);
QueryOutput output = new QueryAction().execute(input); QueryOutput output = new QueryAction().execute(input);
runBackendStepOutput.setRecords(output.getRecords()); runBackendStepOutput.setRecords(output.getRecords());

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java
View File

@ -50,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.commons.lang.BooleanUtils;
/******************************************************************************* /*******************************************************************************
@ -87,9 +88,10 @@ public class StoreSavedBulkLoadProfileProcess implements BackendStep
try try
{ {
String userId = QContext.getQSession().getUser().getIdReference(); String userId = QContext.getQSession().getUser().getIdReference();
String tableName = runBackendStepInput.getValueString("tableName"); String tableName = runBackendStepInput.getValueString("tableName");
String label = runBackendStepInput.getValueString("label"); String label = runBackendStepInput.getValueString("label");
Boolean isBulkEdit = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("isBulkEdit"));
String mappingJson = processMappingJson(runBackendStepInput.getValueString("mappingJson")); String mappingJson = processMappingJson(runBackendStepInput.getValueString("mappingJson"));
@ -98,6 +100,7 @@ public class StoreSavedBulkLoadProfileProcess implements BackendStep
.withValue("mappingJson", mappingJson) .withValue("mappingJson", mappingJson)
.withValue("label", label) .withValue("label", label)
.withValue("tableName", tableName) .withValue("tableName", tableName)
.withValue("isBulkEdit", isBulkEdit)
.withValue("userId", userId); .withValue("userId", userId);
List<QRecord> savedBulkLoadProfileList; List<QRecord> savedBulkLoadProfileList;

18
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java
View File

@ -38,7 +38,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
/******************************************************************************* /*******************************************************************************
** Unit test for BulkLoadMappingSuggester ** Unit test for BulkLoadMappingSuggester
*******************************************************************************/ *******************************************************************************/
class BulkLoadMappingSuggesterTest extends BaseTest class BulkLoadMappingSuggesterTest extends BaseTest
{ {
@ -52,7 +52,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY);
List<String> headerRow = List.of("Id", "First Name", "lastname", "email", "homestate"); List<String> headerRow = List.of("Id", "First Name", "lastname", "email", "homestate");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false);
assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("v1", bulkLoadProfile.getVersion());
assertEquals("FLAT", bulkLoadProfile.getLayout()); assertEquals("FLAT", bulkLoadProfile.getLayout());
assertNull(getFieldByName(bulkLoadProfile, "id")); assertNull(getFieldByName(bulkLoadProfile, "id"));
@ -73,7 +73,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "shipto name", "sku", "quantity"); List<String> headerRow = List.of("orderNo", "shipto name", "sku", "quantity");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false);
assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("v1", bulkLoadProfile.getVersion());
assertEquals("TALL", bulkLoadProfile.getLayout()); assertEquals("TALL", bulkLoadProfile.getLayout());
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
@ -93,7 +93,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity"); List<String> headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false);
assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("v1", bulkLoadProfile.getVersion());
assertEquals("TALL", bulkLoadProfile.getLayout()); assertEquals("TALL", bulkLoadProfile.getLayout());
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
@ -120,7 +120,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); List<String> headerRow = List.of("orderNo", "ship to name", "address 1", "address 2");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false);
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex());
@ -136,7 +136,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); List<String> headerRow = List.of("orderNo", "ship to name", "address 1", "address 2");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false);
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex()); assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex());
@ -152,7 +152,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "ship to name", "address", "address 2"); List<String> headerRow = List.of("orderNo", "ship to name", "address", "address 2");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false);
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex());
@ -177,7 +177,7 @@ class BulkLoadMappingSuggesterTest extends BaseTest
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2"); List<String> headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow, false);
assertEquals("v1", bulkLoadProfile.getVersion()); assertEquals("v1", bulkLoadProfile.getVersion());
assertEquals("WIDE", bulkLoadProfile.getLayout()); assertEquals("WIDE", bulkLoadProfile.getLayout());
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
@ -206,4 +206,4 @@ class BulkLoadMappingSuggesterTest extends BaseTest
.findFirst().orElse(null)); .findFirst().orElse(null));
} }
} }

2
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -1932,6 +1932,8 @@ public class QJavalinImplementation
input.setPossibleValueSourceName(possibleValueSourceName); input.setPossibleValueSourceName(possibleValueSourceName);
input.setSearchTerm(searchTerm); input.setSearchTerm(searchTerm);
input.setDefaultQueryFilter(defaultFilter); input.setDefaultQueryFilter(defaultFilter);
input.setPathParamMap(context.pathParamMap());
input.setQueryParamMap(context.queryParamMap());
if(StringUtils.hasContent(ids)) if(StringUtils.hasContent(ids))
{ {