QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-21 22:48:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 - Bulk load checkpoint:

Browse Source 
- Switch wide format to identify associations via comma-number-indexes...
- Add suggested mappings
- use header name instead of column index for mappings
- add counts of children process summary lines
- excel value/type handling
This commit is contained in:
darin.kelkhoff
2024-11-25 10:07:26 -06:00
parent 9ad9d52634
commit 58ae17bbac
29 changed files with 1167 additions and 988 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -73,7 +73,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.Bulk
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep;
@ -81,7 +80,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager;
@ -815,75 +813,11 @@ public class QInstanceEnricher
/*******************************************************************************
**
*******************************************************************************/
private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName)
public void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName)
{
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
BulkInsertExtractStep.class,
BulkInsertTransformStep.class,
BulkInsertLoadStep.class,
values
)
.withName(processName)
.withLabel(table.getLabel() + " Bulk Insert")
.withTableName(table.getName())
.withIsHidden(true)
.withPermissionRules(qInstance.getDefaultPermissionRules().clone()
.withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class)));
List<QFieldMetaData> editableFields = new ArrayList<>();
for(QFieldSection section : CollectionUtils.nonNullList(table.getSections()))
{
for(String fieldName : CollectionUtils.nonNullList(section.getFieldNames()))
{
try
{
QFieldMetaData field = table.getField(fieldName);
if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB))
{
editableFields.add(field);
}
}
catch(Exception e)
{
// shrug?
}
}
}
String fieldsForHelpText = editableFields.stream()
.map(QFieldMetaData::getLabel)
.collect(Collectors.joining(", "));
QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData()
.withName("upload")
.withLabel("Upload File")
.withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("previewText", "file upload instructions")
.withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM));
process.addStep(0, uploadScreen);
process.getFrontendStep("review").setRecordListFields(editableFields);
qInstance.addProcess(process);
}
/*******************************************************************************
**
*******************************************************************************/
public void defineTableBulkInsertV2(QInstance qInstance, QTableMetaData table, String processName)
{
qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum("bulkInsertFileLayout", BulkInsertMapping.Layout.values()));
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
BulkInsertV2ExtractStep.class,
BulkInsertTransformStep.class,

128
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java
View File

@ -25,27 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadMappingSuggester;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -62,7 +54,26 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput);
buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput);
String tableName = runBackendStepInput.getValueString("tableName");
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName);
runBackendStepOutput.addValue("tableStructure", tableStructure);
@SuppressWarnings("unchecked")
List<String> headerValues = (List<String>) runBackendStepOutput.getValue("headerValues");
buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput);
}
/***************************************************************************
**
***************************************************************************/
private void buildSuggestedMapping(List<String> headerValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput)
{
BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester();
BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues);
runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile);
}
@ -121,6 +132,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
}
}
runBackendStepOutput.addValue("bodyValuesPreview", bodyValues);
}
catch(Exception e)
{
@ -147,94 +159,4 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
return (rs.toString());
}
/***************************************************************************
**
***************************************************************************/
private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
String tableName = runBackendStepInput.getValueString("tableName");
BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null);
runBackendStepOutput.addValue("tableStructure", tableStructure);
}
/***************************************************************************
**
***************************************************************************/
private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath)
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
BulkLoadTableStructure tableStructure = new BulkLoadTableStructure();
tableStructure.setTableName(tableName);
tableStructure.setLabel(table.getLabel());
Set<String> associationJoinFieldNamesToExclude = new HashSet<>();
if(association == null)
{
tableStructure.setIsMain(true);
tableStructure.setIsMany(false);
tableStructure.setAssociationPath(null);
}
else
{
tableStructure.setIsMain(false);
QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName());
if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE))
{
tableStructure.setIsMany(true);
}
for(JoinOn joinOn : join.getJoinOns())
{
////////////////////////////////////////////////////////////////////////////////////////////////
// don't allow the user to map the "join field" from a child up to its parent //
// (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) //
////////////////////////////////////////////////////////////////////////////////////////////////
if(join.getLeftTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getLeftField());
}
else if(join.getRightTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getRightField());
}
}
if(!StringUtils.hasContent(parentAssociationPath))
{
tableStructure.setAssociationPath(association.getName());
}
else
{
tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName());
}
}
ArrayList<QFieldMetaData> fields = new ArrayList<>();
tableStructure.setFields(fields);
for(QFieldMetaData field : table.getFields().values())
{
if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName()))
{
fields.add(field);
}
}
fields.sort(Comparator.comparing(f -> f.getLabel()));
for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations()))
{
BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath);
tableStructure.addAssociation(associatedStructure);
}
return (tableStructure);
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java
View File

@ -51,7 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java
View File

@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
@ -102,7 +102,12 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
BulkLoadFileRow headerRow = fileToRowsInterface.next();
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getColumnIndex() != null)
if(bulkLoadProfileField.getHeaderName() != null)
{
String headerName = bulkLoadProfileField.getHeaderName();
fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName);
}
else if(bulkLoadProfileField.getColumnIndex() != null)
{
String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex()));
fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName);
@ -164,7 +169,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
{
if(bulkLoadProfileField.getFieldName().contains("."))
{
associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.')));
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// handle parent.child.grandchild.fieldName,index.index.index if we do sub-indexes for grandchildren... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
String fieldNameBeforeIndex = bulkLoadProfileField.getFieldName().split(",")[0];
associationNameSet.add(fieldNameBeforeIndex.substring(0, fieldNameBeforeIndex.lastIndexOf('.')));
}
}
bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet));

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java
View File

@ -32,7 +32,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java
View File

@ -109,6 +109,7 @@ public class BulkInsertStepUtils
BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField();
fieldList.add(bulkLoadProfileField);
bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName"));
bulkLoadProfileField.setHeaderName(jsonObject.has("headerName") ? jsonObject.getString("headerName") : null);
bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null);
bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue"));
bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping"));

29
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
@ -67,7 +68,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted");
private Map<UniqueKey, ProcessSummaryLineWithUKSampleValues> ukErrorSummaries = new HashMap<>();
private Map<UniqueKey, ProcessSummaryLineWithUKSampleValues> ukErrorSummaries = new HashMap<>();
private Map<String, ProcessSummaryLine> associationsToInsertSummaries = new HashMap<>();
private QTableMetaData table;
@ -259,6 +261,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep
{
okSummary.incrementCountAndAddPrimaryKey(null);
outputRecords.add(record);
for(Map.Entry<String, List<QRecord>> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet())
{
String associationName = entry.getKey();
ProcessSummaryLine associationToInsertLine = associationsToInsertSummaries.computeIfAbsent(associationName, x -> new ProcessSummaryLine(Status.OK));
associationToInsertLine.incrementCount(CollectionUtils.nonNullList(entry.getValue()).size());
}
}
}
@ -366,6 +375,24 @@ public class BulkInsertTransformStep extends AbstractTransformStep
okSummary.pickMessage(isForResultScreen);
okSummary.addSelfToListIfAnyCount(rs);
for(Map.Entry<String, ProcessSummaryLine> entry : associationsToInsertSummaries.entrySet())
{
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(entry.getKey())).findFirst();
if(association.isPresent())
{
QTableMetaData associationTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
String associationLabel = associationTable.getLabel();
ProcessSummaryLine line = entry.getValue();
line.setSingularFutureMessage(associationLabel + " record will be inserted.");
line.setPluralFutureMessage(associationLabel + " records will be inserted.");
line.setSingularPastMessage(associationLabel + " record was inserted.");
line.setPluralPastMessage(associationLabel + " records were inserted.");
line.pickMessage(isForResultScreen);
line.addSelfToListIfAnyCount(rs);
}
}
for(Map.Entry<UniqueKey, ProcessSummaryLineWithUKSampleValues> entry : ukErrorSummaries.entrySet())
{
UniqueKey uniqueKey = entry.getKey();

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java
View File

@ -32,8 +32,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;

41
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java
View File

@ -25,9 +25,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.MathContext;
import java.time.LocalDateTime;
import java.util.Optional;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.dhatim.fastexcel.reader.Cell;
import org.dhatim.fastexcel.reader.ReadableWorkbook;
import org.dhatim.fastexcel.reader.Sheet;
@ -74,7 +79,41 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows<org.dhatim.f
for(int i = 0; i < readerRow.getCellCount(); i++)
{
values[i] = readerRow.getCell(i).getText();
Cell cell = readerRow.getCell(i);
if(cell.getType() != null)
{
values[i] = switch(cell.getType())
{
case NUMBER ->
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// ... with fastexcel reader, we don't get styles... so, we just know type = number, for dates and ints & decimals... //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Optional<LocalDateTime> dateTime = readerRow.getCellAsDate(i);
if(dateTime.isPresent() && dateTime.get().getYear() > 1915 && dateTime.get().getYear() < 2100)
{
yield dateTime.get();
}
Optional<BigDecimal> optionalBigDecimal = readerRow.getCellAsNumber(i);
if(optionalBigDecimal.isPresent())
{
BigDecimal bigDecimal = optionalBigDecimal.get();
if(bigDecimal.subtract(bigDecimal.round(new MathContext(0))).compareTo(BigDecimal.ZERO) == 0)
{
yield bigDecimal.intValue();
}
yield bigDecimal;
}
yield (null);
}
case BOOLEAN -> readerRow.getCellAsBoolean(i).orElse(null);
case STRING, FORMULA -> cell.getText();
case EMPTY, ERROR -> null;
};
}
}
return new BulkLoadFileRow(values);

216
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java
View File

@ -1,216 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertWideLayoutMapping
{
private List<ChildRecordMapping> childRecordMappings;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public BulkInsertWideLayoutMapping(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
}
/***************************************************************************
**
***************************************************************************/
public static class ChildRecordMapping
{
Map<String, String> fieldNameToHeaderNameMaps;
Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public ChildRecordMapping(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public ChildRecordMapping(Map<String, String> fieldNameToHeaderNameMaps, Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
}
/*******************************************************************************
** Getter for fieldNameToHeaderNameMaps
*******************************************************************************/
public Map<String, String> getFieldNameToHeaderNameMaps()
{
return (this.fieldNameToHeaderNameMaps);
}
/*******************************************************************************
** Setter for fieldNameToHeaderNameMaps
*******************************************************************************/
public void setFieldNameToHeaderNameMaps(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
}
/*******************************************************************************
** Fluent setter for fieldNameToHeaderNameMaps
*******************************************************************************/
public ChildRecordMapping withFieldNameToHeaderNameMaps(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
return (this);
}
/*******************************************************************************
** Getter for associationNameToChildRecordMappingMap
*******************************************************************************/
public Map<String, BulkInsertWideLayoutMapping> getAssociationNameToChildRecordMappingMap()
{
return (this.associationNameToChildRecordMappingMap);
}
/*******************************************************************************
** Setter for associationNameToChildRecordMappingMap
*******************************************************************************/
public void setAssociationNameToChildRecordMappingMap(Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
}
/*******************************************************************************
** Fluent setter for associationNameToChildRecordMappingMap
*******************************************************************************/
public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
return (this);
}
/***************************************************************************
**
***************************************************************************/
public Map<String, Integer> getFieldIndexes(BulkLoadFileRow headerRow)
{
// todo memoize or otherwise don't recompute
Map<String, Integer> rs = new HashMap<>();
////////////////////////////////////////////////////////
// for the current file, map header values to indexes //
////////////////////////////////////////////////////////
Map<String, Integer> headerToIndexMap = new HashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
headerToIndexMap.put(headerValue, i);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// loop over fields - finding what header name they are mapped to - then what index that header is at. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
for(Map.Entry<String, String> entry : fieldNameToHeaderNameMaps.entrySet())
{
String headerName = entry.getValue();
if(headerName != null)
{
Integer headerIndex = headerToIndexMap.get(headerName);
if(headerIndex != null)
{
rs.put(entry.getKey(), headerIndex);
}
}
}
return (rs);
}
}
/*******************************************************************************
** Getter for childRecordMappings
*******************************************************************************/
public List<ChildRecordMapping> getChildRecordMappings()
{
return (this.childRecordMappings);
}
/*******************************************************************************
** Setter for childRecordMappings
*******************************************************************************/
public void setChildRecordMappings(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
}
/*******************************************************************************
** Fluent setter for childRecordMappings
*******************************************************************************/
public BulkInsertWideLayoutMapping withChildRecordMappings(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
return (this);
}
}

232
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java Normal file
View File

@ -0,0 +1,232 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
/*******************************************************************************
** Given a bulk-upload, create a suggested mapping
*******************************************************************************/
public class BulkLoadMappingSuggester
{
private Map<String, Integer> massagedHeadersWithoutNumbersToIndexMap;
private Map<String, Integer> massagedHeadersWithNumbersToIndexMap;
private String layout = "FLAT";
/***************************************************************************
**
***************************************************************************/
public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List<String> headerRow)
{
massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = massageHeader(headerRow.get(i), true);
if(!massagedHeadersWithoutNumbersToIndexMap.containsKey(headerValue))
{
massagedHeadersWithoutNumbersToIndexMap.put(headerValue, i);
}
}
massagedHeadersWithNumbersToIndexMap = new LinkedHashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = massageHeader(headerRow.get(i), false);
if(!massagedHeadersWithNumbersToIndexMap.containsKey(headerValue))
{
massagedHeadersWithNumbersToIndexMap.put(headerValue, i);
}
}
ArrayList<BulkLoadProfileField> fieldList = new ArrayList<>();
processTable(tableStructure, fieldList, headerRow);
/////////////////////////////////////////////////
// sort the fields to match the column indexes //
/////////////////////////////////////////////////
fieldList.sort(Comparator.comparing(blpf -> blpf.getColumnIndex()));
BulkLoadProfile bulkLoadProfile = new BulkLoadProfile()
.withVersion("v1")
.withLayout(layout)
.withHasHeaderRow(true)
.withFieldList(fieldList);
return (bulkLoadProfile);
}
/***************************************************************************
**
***************************************************************************/
private void processTable(BulkLoadTableStructure tableStructure, ArrayList<BulkLoadProfileField> fieldList, List<String> headerRow)
{
Map<String, Integer> rs = new HashMap<>();
for(QFieldMetaData field : tableStructure.getFields())
{
String fieldName = massageHeader(field.getName(), false);
String fieldLabel = massageHeader(field.getLabel(), false);
String tablePlusFieldLabel = massageHeader(QContext.getQInstance().getTable(tableStructure.getTableName()).getLabel() + ": " + field.getLabel(), false);
String fullFieldName = (StringUtils.hasContent(tableStructure.getAssociationPath()) ? (tableStructure.getAssociationPath() + ".") : "") + field.getName();
////////////////////////////////////////////////////////////////////////////////////
// consider, if this is a many-table, if there are many matches, for wide mode... //
////////////////////////////////////////////////////////////////////////////////////
if(tableStructure.getIsMany())
{
List<Integer> matchingIndexes = new ArrayList<>();
for(Map.Entry<String, Integer> entry : massagedHeadersWithNumbersToIndexMap.entrySet())
{
String header = entry.getKey();
if(header.matches(fieldName + "\\d*$") || header.matches(fieldLabel + "\\d*$"))
{
matchingIndexes.add(entry.getValue());
}
}
if(CollectionUtils.nullSafeHasContents(matchingIndexes))
{
///////////////////////////////////////////////////////////////////////////////////////////////////////
// if we found more than 1 match - consider this a likely wide file, and build fields as wide-fields //
// else, if only 1, allow us to go down into the TALL block below //
// note - should we do a merger at the end, in case we found some wide, some tall? //
///////////////////////////////////////////////////////////////////////////////////////////////////////
if(matchingIndexes.size() > 1)
{
layout = "WIDE";
int i = 0;
for(Integer index : matchingIndexes)
{
fieldList.add(new BulkLoadProfileField()
.withFieldName(fullFieldName + "," + i)
.withHeaderName(headerRow.get(index))
.withColumnIndex(index)
);
i++;
}
continue;
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// else - look for matches, first w/ headers with numbers, then headers w/o numbers checking labels and names //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Integer index = null;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// for each of these potential identities of the field: //
// 1) its label, massaged //
// 2) its name, massaged //
// 3) its label, massaged, with numbers stripped away //
// 4) its name, massaged, with numbers stripped away //
// check if that identity is in the massagedHeadersWithNumbersToIndexMap, or the massagedHeadersWithoutNumbersToIndexMap. //
// this is currently successful in the both versions of the address 1 / address 2 <=> address / address 2 use-case //
// that is, BulkLoadMappingSuggesterTest.testChallengingAddress1And2 //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(String fieldIdentity : ListBuilder.of(fieldLabel, fieldName, tablePlusFieldLabel, massageHeader(fieldLabel, true), massageHeader(fieldName, true)))
{
if(massagedHeadersWithNumbersToIndexMap.containsKey(fieldIdentity))
{
index = massagedHeadersWithNumbersToIndexMap.get(fieldIdentity);
}
else if(massagedHeadersWithoutNumbersToIndexMap.containsKey(fieldIdentity))
{
index = massagedHeadersWithoutNumbersToIndexMap.get(fieldIdentity);
}
if(index != null)
{
break;
}
}
if(index != null)
{
fieldList.add(new BulkLoadProfileField()
.withFieldName(fullFieldName)
.withHeaderName(headerRow.get(index))
.withColumnIndex(index)
);
if(tableStructure.getIsMany() && layout.equals("FLAT"))
{
//////////////////////////////////////////////////////////////////////////////////////////
// the first time we find an is-many child, if we were still marked as flat, go to tall //
//////////////////////////////////////////////////////////////////////////////////////////
layout = "TALL";
}
}
}
////////////////////////////////////////////
// recursively process child associations //
////////////////////////////////////////////
for(BulkLoadTableStructure associationTableStructure : CollectionUtils.nonNullList(tableStructure.getAssociations()))
{
processTable(associationTableStructure, fieldList, headerRow);
}
}
/***************************************************************************
**
***************************************************************************/
private String massageHeader(String header, boolean stripNumbers)
{
if(header == null)
{
return (null);
}
String massagedWithNumbers = header.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", "");
return stripNumbers ? massagedWithNumbers.replaceAll("[0-9]", "") : massagedWithNumbers;
}
}

132
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java Normal file
View File

@ -0,0 +1,132 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** utility to build BulkLoadTableStructure objects for a QQQ Table.
*******************************************************************************/
public class BulkLoadTableStructureBuilder
{
/***************************************************************************
**
***************************************************************************/
public static BulkLoadTableStructure buildTableStructure(String tableName)
{
return (buildTableStructure(tableName, null, null));
}
/***************************************************************************
**
***************************************************************************/
private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath)
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
BulkLoadTableStructure tableStructure = new BulkLoadTableStructure();
tableStructure.setTableName(tableName);
tableStructure.setLabel(table.getLabel());
Set<String> associationJoinFieldNamesToExclude = new HashSet<>();
if(association == null)
{
tableStructure.setIsMain(true);
tableStructure.setIsMany(false);
tableStructure.setAssociationPath(null);
}
else
{
tableStructure.setIsMain(false);
QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName());
if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE))
{
tableStructure.setIsMany(true);
}
for(JoinOn joinOn : join.getJoinOns())
{
////////////////////////////////////////////////////////////////////////////////////////////////
// don't allow the user to map the "join field" from a child up to its parent //
// (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) //
////////////////////////////////////////////////////////////////////////////////////////////////
if(join.getLeftTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getLeftField());
}
else if(join.getRightTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getRightField());
}
}
if(!StringUtils.hasContent(parentAssociationPath))
{
tableStructure.setAssociationPath(association.getName());
}
else
{
tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName());
}
}
ArrayList<QFieldMetaData> fields = new ArrayList<>();
tableStructure.setFields(fields);
for(QFieldMetaData field : table.getFields().values())
{
if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName()))
{
fields.add(field);
}
}
fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), "")));
for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations()))
{
BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath);
tableStructure.addAssociation(associatedStructure);
}
return (tableStructure);
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java
View File

@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;

58
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java
View File

@ -27,9 +27,10 @@ import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -48,42 +49,49 @@ public interface RowsToRecordInterface
/***************************************************************************
** returns true if value from row was used, else false.
***************************************************************************/
default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index)
default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex)
{
String fieldName = field.getName();
QFieldType type = field.getType();
return setValueOrDefault(record, field, associationNameChain, mapping, row, columnIndex, null);
}
boolean valueFromRowWasUsed = false;
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName;
/***************************************************************************
** returns true if value from row was used, else false.
***************************************************************************/
default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex, List<Integer> wideAssociationIndexes)
{
boolean valueFromRowWasUsed = false;
Serializable value = null;
if(index != null && row != null)
/////////////////////////////////////////////////////////////////////////////////////////////////
// build full field-name -- possibly associations, then field name, then possibly index-suffix //
/////////////////////////////////////////////////////////////////////////////////////////////////
String fieldName = field.getName();
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName;
String wideAssociationSuffix = "";
if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes))
{
value = row.getValueElseNull(index);
wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes);
}
String fullFieldName = fieldNameWithAssociationPrefix + wideAssociationSuffix;
//////////////////////////////////////////////
// ok - look in the row - then the defaults //
//////////////////////////////////////////////
Serializable value = null;
if(columnIndex != null && row != null)
{
value = row.getValueElseNull(columnIndex);
if(value != null && !"".equals(value))
{
valueFromRowWasUsed = true;
}
}
else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix))
else if(mapping.getFieldNameToDefaultValueMap().containsKey(fullFieldName))
{
value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix);
value = mapping.getFieldNameToDefaultValueMap().get(fullFieldName);
}
/* note - moving this to ValueMapper...
if(value != null)
{
try
{
value = ValueUtils.getValueAsFieldType(type, value);
}
catch(Exception e)
{
record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]"));
}
}
*/
if(value != null)
{
record.setValue(fieldName, value);

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java
View File

@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java
View File

@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;

196
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java Normal file
View File

@ -0,0 +1,196 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
** use a flatter mapping object, where field names look like:
** associationChain.fieldName,index.subIndex
*******************************************************************************/
public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>());
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
/***************************************************************************
** may return null, if there were no values in the row for this (sub-wide) record.
***************************************************************************/
private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map<String, Integer> fieldIndexes, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes) throws QException
{
//////////////////////////////////////////////////////
// start by building the record with its own fields //
//////////////////////////////////////////////////////
QRecord record = new QRecord();
boolean hadAnyValuesInRow = false;
for(QFieldMetaData field : table.getFields().values())
{
hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow;
}
if(!hadAnyValuesInRow)
{
return (null);
}
/////////////////////////////
// associations (children) //
/////////////////////////////
for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations()))
{
boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName);
if(processAssociation)
{
String associationNameMinusChain = StringUtils.hasContent(associationNameChain)
? associationName.substring(associationNameChain.length() + 1)
: associationName;
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
List<QRecord> associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, row, headerRow);
record.withAssociatedRecords(associationNameMinusChain, associatedRecords);
}
}
return record;
}
/***************************************************************************
**
***************************************************************************/
private List<QRecord> processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException
{
List<QRecord> rs = new ArrayList<>();
String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName;
for(int i = 0; true; i++)
{
// todo - doesn't support grand-children
List<Integer> wideAssociationIndexes = List.of(i);
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes);
if(fieldIndexes.isEmpty())
{
break;
}
QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes);
if(record != null)
{
rs.add(record);
}
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/
boolean shouldProcessAssociation(String associationNameChain, String associationName)
{
return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p ->
{
List<String> chainParts = new ArrayList<>();
List<String> nameParts = new ArrayList<>();
if(StringUtils.hasContent(associationNameChain))
{
chainParts.addAll(Arrays.asList(associationNameChain.split("\\.")));
}
if(StringUtils.hasContent(associationName))
{
nameParts.addAll(Arrays.asList(associationName.split("\\.")));
}
if(!nameParts.isEmpty())
{
nameParts.remove(nameParts.size() - 1);
}
return (chainParts.equals(nameParts));
}).orElse(false);
}
}

260
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java
View File

@ -1,260 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = new QRecord();
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()));
}
processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record);
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private void processAssociations(Map<String, BulkInsertWideLayoutMapping> mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException
{
for(Map.Entry<String, BulkInsertWideLayoutMapping> entry : CollectionUtils.nonNullMap(mappingMap).entrySet())
{
String associationName = entry.getKey();
BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue();
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName;
for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings())
{
QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow);
if(associatedRecord != null)
{
record.withAssociatedRecord(associationName, associatedRecord);
}
}
}
}
/***************************************************************************
**
***************************************************************************/
private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException
{
Map<String, Integer> fieldIndexes = childRecordMapping.getFieldIndexes(headerRow);
QRecord associatedRecord = new QRecord();
boolean usedAnyValuesFromRow = false;
for(QFieldMetaData field : table.getFields().values())
{
boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
usedAnyValuesFromRow |= valueFromRowWasUsed;
}
if(usedAnyValuesFromRow)
{
processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord);
return (associatedRecord);
}
else
{
return (null);
}
}
// /***************************************************************************
// **
// ***************************************************************************/
// private List<QRecord> processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException
// {
// List<QRecord> rs = new ArrayList<>();
// Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
// for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
// {
// if(entry.getKey().startsWith(associationName + "."))
// {
// String fieldName = entry.getKey().substring(associationName.length() + 1);
// //////////////////////////////////////////////////////////////////////////
// // make sure the name here is for this table - not a sub-table under it //
// //////////////////////////////////////////////////////////////////////////
// if(!fieldName.contains("."))
// {
// fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue());
// }
// }
// }
// /////////////////////////////////////////////////////////////////////
// // loop over the length of the record, building associated records //
// /////////////////////////////////////////////////////////////////////
// QRecord associatedRecord = new QRecord();
// Set<String> processedFieldNames = new HashSet<>();
// boolean gotAnyValues = false;
// int subStartIndex = -1;
// for(int i = startIndex; i < endIndex; i++)
// {
// String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
// {
// ///////////////////////////////////////////////
// // ok - this is a value for this association //
// ///////////////////////////////////////////////
// if(subStartIndex == -1)
// {
// subStartIndex = i;
// }
// String fieldName = entry.getKey();
// if(processedFieldNames.contains(fieldName))
// {
// /////////////////////////////////////////////////
// // this means we're starting a new sub-record! //
// /////////////////////////////////////////////////
// if(gotAnyValues)
// {
// addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
// processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i);
// rs.add(associatedRecord);
// }
// associatedRecord = new QRecord();
// processedFieldNames = new HashSet<>();
// gotAnyValues = false;
// subStartIndex = i + 1;
// }
// processedFieldNames.add(fieldName);
// Serializable value = row.getValueElseNull(i);
// if(value != null && !"".equals(value))
// {
// gotAnyValues = true;
// }
// setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i);
// }
// }
// }
// ////////////////////////
// // handle final value //
// ////////////////////////
// if(gotAnyValues)
// {
// addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
// processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex);
// rs.add(associatedRecord);
// }
// return (rs);
// }
/***************************************************************************
**
***************************************************************************/
private void addDefaultValuesToAssociatedRecord(Set<String> processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain)
{
for(QFieldMetaData field : table.getFields().values())
{
if(!processedFieldNames.contains(field.getName()))
{
setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null);
}
}
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java
View File

@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java
View File

@ -19,7 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable;
@ -34,7 +34,10 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.FlatRowsToRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.TallRowsToRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -63,8 +66,7 @@ public class BulkInsertMapping implements Serializable
private Map<String, Serializable> fieldNameToDefaultValueMap = new HashMap<>();
private Map<String, Map<String, Serializable>> fieldNameToValueMapping = new HashMap<>();
private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>();
private Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping = new HashMap<>();
private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>();
private List<String> mappedAssociations = new ArrayList<>();
@ -79,7 +81,7 @@ public class BulkInsertMapping implements Serializable
{
FLAT(FlatRowsToRecord::new),
TALL(TallRowsToRecord::new),
WIDE(WideRowsToRecordWithExplicitMapping::new);
WIDE(WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping::new);
/***************************************************************************
@ -137,10 +139,21 @@ public class BulkInsertMapping implements Serializable
***************************************************************************/
@JsonIgnore
public Map<String, Integer> getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException
{
return getFieldIndexes(table, associationNameChain, headerRow, null);
}
/***************************************************************************
**
***************************************************************************/
@JsonIgnore
public Map<String, Integer> getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes) throws QException
{
if(hasHeaderRow && fieldNameToHeaderNameMap != null)
{
return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow));
return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow, wideAssociationIndexes));
}
else if(fieldNameToIndexMap != null)
{
@ -208,7 +221,7 @@ public class BulkInsertMapping implements Serializable
/***************************************************************************
**
***************************************************************************/
private Map<String, Integer> getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow)
private Map<String, Integer> getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes)
{
Map<String, Integer> rs = new HashMap<>();
@ -222,13 +235,19 @@ public class BulkInsertMapping implements Serializable
headerToIndexMap.put(headerValue, i);
}
String wideAssociationSuffix = "";
if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes))
{
wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// loop over fields - finding what header name they are mapped to - then what index that header is at. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + ".";
for(QFieldMetaData field : table.getFields().values())
{
String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName());
String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix);
if(headerName != null)
{
Integer headerIndex = headerToIndexMap.get(headerName);
@ -527,34 +546,4 @@ public class BulkInsertMapping implements Serializable
}
/*******************************************************************************
** Getter for wideLayoutMapping
*******************************************************************************/
public Map<String, BulkInsertWideLayoutMapping> getWideLayoutMapping()
{
return (this.wideLayoutMapping);
}
/*******************************************************************************
** Setter for wideLayoutMapping
*******************************************************************************/
public void setWideLayoutMapping(Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping)
{
this.wideLayoutMapping = wideLayoutMapping;
}
/*******************************************************************************
** Fluent setter for wideLayoutMapping
*******************************************************************************/
public BulkInsertMapping withWideLayoutMapping(Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping)
{
this.wideLayoutMapping = wideLayoutMapping;
return (this);
}
}

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java
View File

@ -33,6 +33,7 @@ public class BulkLoadProfileField
{
private String fieldName;
private Integer columnIndex;
private String headerName;
private Serializable defaultValue;
private Boolean doValueMapping;
private Map<String, Serializable> valueMappings;
@ -192,4 +193,35 @@ public class BulkLoadProfileField
return (this);
}
/*******************************************************************************
** Getter for headerName
*******************************************************************************/
public String getHeaderName()
{
return (this.headerName);
}
/*******************************************************************************
** Setter for headerName
*******************************************************************************/
public void setHeaderName(String headerName)
{
this.headerName = headerName;
}
/*******************************************************************************
** Fluent setter for headerName
*******************************************************************************/
public BulkLoadProfileField withHeaderName(String headerName)
{
this.headerName = headerName;
return (this);
}
}