QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 Checkpoint on bulk-load backend

Browse Source
This commit is contained in:
darin.kelkhoff
2024-11-19 08:44:43 -06:00
parent b684f2409b
commit d8ac14a756
35 changed files with 2682 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java Normal file
View File

@ -0,0 +1,240 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertPrepareFileMappingStep implements BackendStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput);
buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput);
}
/***************************************************************************
**
***************************************************************************/
private static void buildFileDetailsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
File file = new File(storageInput.getReference());
runBackendStepOutput.addValue("fileBaseName", file.getName());
try
(
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// open a stream to read from our file, and a FileToRows object, that knows how to read from that stream //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
/////////////////////////////////////////////////
// read the 1st row, and assume it is a header //
/////////////////////////////////////////////////
BulkLoadFileRow headerRow = fileToRowsInterface.next();
ArrayList<String> headerValues = new ArrayList<>();
ArrayList<String> headerLetters = new ArrayList<>();
for(int i = 0; i < headerRow.size(); i++)
{
headerValues.add(ValueUtils.getValueAsString(headerRow.getValue(i)));
headerLetters.add(toHeaderLetter(i));
}
runBackendStepOutput.addValue("headerValues", headerValues);
runBackendStepOutput.addValue("headerLetters", headerLetters);
///////////////////////////////////////////////////////////////////////////////////////////
// while there are more rows in the file - and we're under preview-rows limit, read rows //
///////////////////////////////////////////////////////////////////////////////////////////
int previewRows = 0;
int previewRowsLimit = 5;
ArrayList<ArrayList<String>> bodyValues = new ArrayList<>();
for(int i = 0; i < headerRow.size(); i++)
{
bodyValues.add(new ArrayList<>());
}
while(fileToRowsInterface.hasNext() && previewRows < previewRowsLimit)
{
BulkLoadFileRow bodyRow = fileToRowsInterface.next();
previewRows++;
for(int i = 0; i < headerRow.size(); i++)
{
bodyValues.get(i).add(ValueUtils.getValueAsString(bodyRow.getValueElseNull(i)));
}
}
runBackendStepOutput.addValue("bodyValuesPreview", bodyValues);
}
catch(Exception e)
{
throw (new QException("Error reading bulk load file", e));
}
}
/***************************************************************************
**
***************************************************************************/
static String toHeaderLetter(int i)
{
StringBuilder rs = new StringBuilder();
do
{
rs.insert(0, (char) ('A' + (i % 26)));
i = (i / 26) - 1;
}
while(i >= 0);
return (rs.toString());
}
/***************************************************************************
**
***************************************************************************/
private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
String tableName = runBackendStepInput.getValueString("tableName");
BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null);
runBackendStepOutput.addValue("tableStructure", tableStructure);
}
/***************************************************************************
**
***************************************************************************/
private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath)
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
BulkLoadTableStructure tableStructure = new BulkLoadTableStructure();
tableStructure.setTableName(tableName);
tableStructure.setLabel(table.getLabel());
Set<String> associationJoinFieldNamesToExclude = new HashSet<>();
if(association == null)
{
tableStructure.setIsMain(true);
tableStructure.setIsMany(false);
tableStructure.setAssociationPath(null);
}
else
{
tableStructure.setIsMain(false);
QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName());
if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE))
{
tableStructure.setIsMany(true);
}
for(JoinOn joinOn : join.getJoinOns())
{
////////////////////////////////////////////////////////////////////////////////////////////////
// don't allow the user to map the "join field" from a child up to its parent //
// (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) //
////////////////////////////////////////////////////////////////////////////////////////////////
if(join.getLeftTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getLeftField());
}
else if(join.getRightTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getRightField());
}
}
if(!StringUtils.hasContent(parentAssociationPath))
{
tableStructure.setAssociationPath(association.getName());
}
else
{
tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName());
}
}
ArrayList<QFieldMetaData> fields = new ArrayList<>();
tableStructure.setFields(fields);
for(QFieldMetaData field : table.getFields().values())
{
if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName()))
{
fields.add(field);
}
}
fields.sort(Comparator.comparing(f -> f.getLabel()));
for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations()))
{
BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath);
tableStructure.addAssociation(associatedStructure);
}
return (tableStructure);
}
}

250
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java Normal file
View File

@ -0,0 +1,250 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.google.gson.reflect.TypeToken;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertPrepareValueMappingStep implements BackendStep
{
private static final QLogger LOG = QLogger.getLogger(BulkInsertPrepareValueMappingStep.class);
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
try
{
/////////////////////////////////////////////////////////////
// prep the frontend for what field we're going to map now //
/////////////////////////////////////////////////////////////
List<String> fieldNamesToDoValueMapping = (List<String>) runBackendStepInput.getValue("fieldNamesToDoValueMapping");
Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex");
if(valueMappingFieldIndex == null)
{
valueMappingFieldIndex = 0;
}
else
{
valueMappingFieldIndex++;
}
////////////////////////////////////////////////////////////////////////////////////////////////////
// if there are no more fields (values) to map, then proceed to the standard streamed-ETL preview //
////////////////////////////////////////////////////////////////////////////////////////////////////
if(valueMappingFieldIndex >= fieldNamesToDoValueMapping.size())
{
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
return;
}
runBackendStepInput.addValue("valueMappingFieldIndex", valueMappingFieldIndex);
String fullFieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex);
TableAndField tableAndField = getTableAndField(runBackendStepInput.getValueString("tableName"), fullFieldName);
runBackendStepInput.addValue("valueMappingField", new QFrontendFieldMetaData(tableAndField.field()));
runBackendStepInput.addValue("valueMappingFullFieldName", fullFieldName);
runBackendStepInput.addValue("valueMappingFieldTableName", tableAndField.table().getName());
////////////////////////////////////////////////////
// get all the values from the file in this field //
// todo - should do all mapping fields at once? //
////////////////////////////////////////////////////
ArrayList<Serializable> fileValues = getValuesForField(tableAndField.table(), tableAndField.field(), fullFieldName, runBackendStepInput);
runBackendStepOutput.addValue("fileValues", fileValues);
///////////////////////////////////////////////
// clear these in case not getting set below //
///////////////////////////////////////////////
runBackendStepOutput.addValue("valueMapping", new HashMap<>());
runBackendStepOutput.addValue("mappedValueLabels", new HashMap<>());
BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping");
HashMap<String, Serializable> valueMapping = null;
if(bulkInsertMapping.getFieldNameToValueMapping() != null && bulkInsertMapping.getFieldNameToValueMapping().containsKey(fullFieldName))
{
valueMapping = CollectionUtils.useOrWrap(bulkInsertMapping.getFieldNameToValueMapping().get(fullFieldName), new TypeToken<>() {});
runBackendStepOutput.addValue("valueMapping", valueMapping);
if(StringUtils.hasContent(tableAndField.field().getPossibleValueSourceName()))
{
HashMap<Serializable, String> possibleValueLabels = loadPossibleValues(tableAndField.field(), valueMapping);
runBackendStepOutput.addValue("mappedValueLabels", possibleValueLabels);
}
}
}
catch(Exception e)
{
LOG.warn("Error in bulk insert prepare value mapping", e);
throw new QException("Unhandled error in bulk insert prepare value mapping step", e);
}
}
/***************************************************************************
**
***************************************************************************/
public static TableAndField getTableAndField(String tableName, String fullFieldName) throws QException
{
List<String> parts = new ArrayList<>(List.of(fullFieldName.split("\\.")));
String fieldBaseName = parts.remove(parts.size() - 1);
QTableMetaData table = QContext.getQInstance().getTable(tableName);
for(String associationName : parts)
{
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isPresent())
{
table = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
}
else
{
throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]");
}
}
TableAndField result = new TableAndField(table, table.getField(fieldBaseName));
return result;
}
/***************************************************************************
**
***************************************************************************/
public record TableAndField(QTableMetaData table, QFieldMetaData field) {}
/***************************************************************************
**
***************************************************************************/
private HashMap<Serializable, String> loadPossibleValues(QFieldMetaData field, Map<String, Serializable> valueMapping) throws QException
{
SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput();
input.setPossibleValueSourceName(field.getPossibleValueSourceName());
input.setIdList(new ArrayList<>(new HashSet<>(valueMapping.values()))); // go through a set to strip dupes
SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input);
HashMap<Serializable, String> rs = new HashMap<>();
for(QPossibleValue<?> result : output.getResults())
{
Serializable id = (Serializable) result.getId();
rs.put(id, result.getLabel());
}
return rs;
}
/***************************************************************************
**
***************************************************************************/
private ArrayList<Serializable> getValuesForField(QTableMetaData table, QFieldMetaData field, String fullFieldName, RunBackendStepInput runBackendStepInput) throws QException
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping");
String associationNameChain = null;
if(fullFieldName.contains("."))
{
associationNameChain = fullFieldName.substring(0, fullFieldName.lastIndexOf('.'));
}
try
(
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
Set<String> values = new LinkedHashSet<>();
BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null;
Map<String, Integer> fieldIndexes = bulkInsertMapping.getFieldIndexes(table, associationNameChain, headerRow);
int index = fieldIndexes.get(field.getName());
while(fileToRowsInterface.hasNext())
{
BulkLoadFileRow row = fileToRowsInterface.next();
Serializable value = row.getValueElseNull(index);
if(value != null)
{
values.add(ValueUtils.getValueAsString(value));
}
if(values.size() > 100)
{
throw (new QUserFacingException("Too many unique values were found for mapping for field: " + field.getName()));
}
}
return (new ArrayList<>(values));
}
catch(Exception e)
{
throw (new QException("Error getting values from file", e));
}
}
}

200
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java Normal file
View File

@ -0,0 +1,200 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveFileMappingStep implements BackendStep
{
private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveFileMappingStep.class);
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
try
{
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
/////////////////////////////////////////////////////////////////////////
// put the list of bulk load profile into the process state - it's the //
// thing that the frontend will be looking at as the saved profile //
/////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile);
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// now build the mapping object that the backend wants - based on the bulkLoadProfile from the frontend //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertMapping bulkInsertMapping = new BulkInsertMapping();
bulkInsertMapping.setTableName(runBackendStepInput.getTableName());
bulkInsertMapping.setHasHeaderRow(bulkLoadProfile.getHasHeaderRow());
bulkInsertMapping.setLayout(BulkInsertMapping.Layout.valueOf(bulkLoadProfile.getLayout()));
//////////////////////////////////////////////////////////////////////////////////////////////
// handle field to name or index mappings (depending on if there's a header row being used) //
//////////////////////////////////////////////////////////////////////////////////////////////
if(BooleanUtils.isTrue(bulkInsertMapping.getHasHeaderRow()))
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
try
(
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
Map<String, String> fieldNameToHeaderNameMap = new HashMap<>();
bulkInsertMapping.setFieldNameToHeaderNameMap(fieldNameToHeaderNameMap);
BulkLoadFileRow headerRow = fileToRowsInterface.next();
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getColumnIndex() != null)
{
String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex()));
fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName);
}
}
}
}
else
{
Map<String, Integer> fieldNameToIndexMap = new HashMap<>();
bulkInsertMapping.setFieldNameToIndexMap(fieldNameToIndexMap);
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getColumnIndex() != null)
{
fieldNameToIndexMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getColumnIndex());
}
}
}
/////////////////////////////////////
// do fields w/ default values now //
/////////////////////////////////////
HashMap<String, Serializable> fieldNameToDefaultValueMap = new HashMap<>();
bulkInsertMapping.setFieldNameToDefaultValueMap(fieldNameToDefaultValueMap);
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getDefaultValue() != null)
{
fieldNameToDefaultValueMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getDefaultValue());
}
}
/////////////////////////////////////////////////////////////////////////////////////////////
// frontend at this point will have sent just told us which field names need value mapping //
// store those - and let them drive the value-mapping screens that we'll go through next //
// todo - uh, what if those come from profile, dummy!?
/////////////////////////////////////////////////////////////////////////////////////////////
ArrayList<String> fieldNamesToDoValueMapping = new ArrayList<>();
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()))
{
fieldNamesToDoValueMapping.add(bulkLoadProfileField.getFieldName());
if(CollectionUtils.nullSafeHasContents(bulkLoadProfileField.getValueMappings()))
{
bulkInsertMapping.getFieldNameToValueMapping().put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getValueMappings());
}
}
}
runBackendStepOutput.addValue("fieldNamesToDoValueMapping", new ArrayList<>(fieldNamesToDoValueMapping));
///////////////////////////////////////////////////////////////////////////////////////
// figure out what associations are being mapped, by looking at the full field names //
///////////////////////////////////////////////////////////////////////////////////////
Set<String> associationNameSet = new HashSet<>();
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getFieldName().contains("."))
{
associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.')));
}
}
bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet));
/////////////////////////////////////////////////////////////////////////////////////////////////////
// at this point we're done populating the bulkInsertMapping object. put it in the process state. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping))
{
//////////////////////////////////////////////////////////////////////////////////
// just go to the prepareValueMapping backend step - it'll figure out the rest. //
// it's also where the value-mapping loop of steps points. //
// and, this will actually be the default (e.g., the step after this one). //
//////////////////////////////////////////////////////////////////////////////////
}
else
{
//////////////////////////////////////////////////////////////////////////////////
// else - if no values to map - continue with the standard streamed-ETL preview //
//////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
}
}
catch(Exception e)
{
LOG.warn("Error in bulk insert receive mapping", e);
throw new QException("Unhandled error in bulk insert receive mapping step", e);
}
}
}

81
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java
View File

@ -1,81 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveFileStep implements BackendStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
try
(
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
BulkLoadFileRow headerRow = fileToRowsInterface.next();
List<String> bodyRows = new ArrayList<>();
while(fileToRowsInterface.hasNext() && bodyRows.size() < 20)
{
bodyRows.add(fileToRowsInterface.next().toString());
}
runBackendStepOutput.addValue("header", headerRow.toString());
runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows));
System.out.println("Done receiving file");
}
catch(QException qe)
{
throw qe;
}
catch(Exception e)
{
throw new QException("Unhandled error in bulk insert extract step", e);
}
}
}

105
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java Normal file
View File

@ -0,0 +1,105 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.core.type.TypeReference;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveValueMappingStep implements BackendStep
{
private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveValueMappingStep.class);
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
try
{
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
List<String> fieldNamesToDoValueMapping = (List<String>) runBackendStepInput.getValue("fieldNamesToDoValueMapping");
Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex");
String fieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex);
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
/////////////////////////////////////////////////////////////////////////
// put the list of bulk load profile into the process state - it's the //
// thing that the frontend will be looking at as the saved profile //
/////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// get the bulkInsertMapping object from the process, creating a fieldNameToValueMapping map within it if needed //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping");
Map<String, Map<String, Serializable>> fieldNameToValueMapping = bulkInsertMapping.getFieldNameToValueMapping();
if(fieldNameToValueMapping == null)
{
fieldNameToValueMapping = new HashMap<>();
bulkInsertMapping.setFieldNameToValueMapping(fieldNameToValueMapping);
}
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
////////////////////////////////////////////////
// put the mapped values into the mapping map //
////////////////////////////////////////////////
Map<String, Serializable> mappedValues = JsonUtils.toObject(runBackendStepInput.getValueString("mappedValuesJSON"), new TypeReference<>() {});
fieldNameToValueMapping.put(fieldName, mappedValues);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
// always return to the prepare-mapping step - as it will determine if it's time to break the loop or not. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepPrepareValueMapping(runBackendStepOutput);
}
catch(Exception e)
{
LOG.warn("Error in bulk insert receive mapping", e);
throw new QException("Unhandled error in bulk insert receive mapping step", e);
}
}
}

96
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java
View File

@ -22,10 +22,22 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.json.JSONArray;
import org.json.JSONObject;
/*******************************************************************************
@ -55,4 +67,88 @@ public class BulkInsertStepUtils
return (storageInput);
}
/***************************************************************************
**
***************************************************************************/
public static void setNextStepStreamedETLPreview(RunBackendStepOutput runBackendStepOutput)
{
runBackendStepOutput.setOverrideLastStepName("receiveValueMapping");
}
/***************************************************************************
**
***************************************************************************/
public static void setNextStepPrepareValueMapping(RunBackendStepOutput runBackendStepOutput)
{
runBackendStepOutput.setOverrideLastStepName("receiveFileMapping");
}
/***************************************************************************
**
***************************************************************************/
public static BulkLoadProfile getBulkLoadProfile(RunBackendStepInput runBackendStepInput)
{
String version = runBackendStepInput.getValueString("version");
if("v1".equals(version))
{
String layout = runBackendStepInput.getValueString("layout");
Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow");
ArrayList<BulkLoadProfileField> fieldList = new ArrayList<>();
JSONArray array = new JSONArray(runBackendStepInput.getValueString("fieldListJSON"));
for(int i = 0; i < array.length(); i++)
{
JSONObject jsonObject = array.getJSONObject(i);
BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField();
fieldList.add(bulkLoadProfileField);
bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName"));
bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null);
bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue"));
bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping"));
if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings"))
{
bulkLoadProfileField.setValueMappings(new HashMap<>());
JSONObject valueMappingsJsonObject = jsonObject.getJSONObject("valueMappings");
for(String fileValue : valueMappingsJsonObject.keySet())
{
bulkLoadProfileField.getValueMappings().put(fileValue, ValueUtils.getValueAsString(valueMappingsJsonObject.get(fileValue)));
}
}
}
BulkLoadProfile bulkLoadProfile = new BulkLoadProfile()
.withFieldList(fieldList)
.withHasHeaderRow(hasHeaderRow)
.withLayout(layout);
return (bulkLoadProfile);
}
else
{
throw (new IllegalArgumentException("Unexpected version for bulk load profile: " + version));
}
}
/***************************************************************************
**
***************************************************************************/
public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId");
if(savedBulkLoadProfileId != null)
{
QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId);
runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord);
}
}
}

68
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -111,6 +111,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep
// since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue(LoadViaInsertStep.FIELD_SKIP_UNIQUE_KEY_CHECK, true);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
// make sure that if a saved profile was selected on a review screen, that the result screen knows about it. //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
}
@ -121,8 +126,43 @@ public class BulkInsertTransformStep extends AbstractTransformStep
@Override
public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
int rowsInThisPage = runBackendStepInput.getRecords().size();
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
int recordsInThisPage = runBackendStepInput.getRecords().size();
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
// split the records w/o UK errors into those w/ e
List<QRecord> recordsWithoutAnyErrors = new ArrayList<>();
List<QRecord> recordsWithSomeErrors = new ArrayList<>();
for(QRecord record : runBackendStepInput.getRecords())
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
recordsWithSomeErrors.add(record);
}
else
{
recordsWithoutAnyErrors.add(record);
}
}
//////////////////////////////////////////////////////////////////
// propagate errors that came into this step out to the summary //
//////////////////////////////////////////////////////////////////
if(!recordsWithSomeErrors.isEmpty())
{
for(QRecord record : recordsWithSomeErrors)
{
String message = record.getErrors().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addError(message, null);
}
}
if(recordsWithoutAnyErrors.isEmpty())
{
////////////////////////////////////////////////////////////////////////////////
// skip th rest of this method if there aren't any records w/o errors in them //
////////////////////////////////////////////////////////////////////////////////
this.rowsProcessed += recordsInThisPage;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations //
@ -130,7 +170,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
InsertInput insertInput = new InsertInput();
insertInput.setInputSource(QInputSource.USER);
insertInput.setTableName(runBackendStepInput.getTableName());
insertInput.setRecords(runBackendStepInput.getRecords());
insertInput.setRecords(recordsWithoutAnyErrors);
insertInput.setSkipUniqueKeyCheck(true);
//////////////////////////////////////////////////////////////////////
@ -145,7 +185,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true);
if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun))
{
List<QRecord> recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, runBackendStepInput.getRecords(), true);
List<QRecord> recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, recordsWithoutAnyErrors, true);
runBackendStepInput.setRecords(recordsAfterCustomizer);
///////////////////////////////////////////////////////////////////////////////////////
@ -159,13 +199,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep
List<UniqueKey> uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys());
for(UniqueKey uniqueKey : uniqueKeys)
{
existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet());
existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, recordsWithoutAnyErrors, uniqueKey).keySet());
ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// on the validate step, we haven't read the full file, so we don't know how many rows there are - thus //
// record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
// todo - move this up (before the early return?) //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
{
@ -187,12 +228,12 @@ public class BulkInsertTransformStep extends AbstractTransformStep
// Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction //
// will only be getting the records in pages, but in here, we'll track UK's across pages!! //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table);
List<QRecord> recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(recordsWithoutAnyErrors, existingKeys, uniqueKeys, table);
/////////////////////////////////////////////////////////////////////////////////
// run all validation from the insert action - in Preview mode (boolean param) //
/////////////////////////////////////////////////////////////////////////////////
insertInput.setRecords(recordsWithoutUkErrors);
insertInput.setRecords(recordsWithoutAnyErrors);
InsertAction insertAction = new InsertAction();
insertAction.performValidations(insertInput, true);
List<QRecord> validationResultRecords = insertInput.getRecords();
@ -222,8 +263,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
}
runBackendStepOutput.setRecords(outputRecords);
this.rowsProcessed += rowsInThisPage;
this.rowsProcessed += recordsInThisPage;
}
@ -231,7 +271,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
/*******************************************************************************
**
*******************************************************************************/
private List<QRecord> getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map<UniqueKey, Set<List<Serializable>>> existingKeys, List<UniqueKey> uniqueKeys, QTableMetaData table)
private List<QRecord> getRecordsWithoutUniqueKeyErrors(List<QRecord> records, Map<UniqueKey, Set<List<Serializable>>> existingKeys, List<UniqueKey> uniqueKeys, QTableMetaData table)
{
////////////////////////////////////////////////////
// if there are no UK's, proceed with all records //
@ -239,7 +279,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
List<QRecord> recordsWithoutUkErrors = new ArrayList<>();
if(existingKeys.isEmpty())
{
recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords());
recordsWithoutUkErrors.addAll(records);
}
else
{
@ -255,7 +295,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
// else, get each records keys and see if it already exists or not //
// also, build a set of keys we've seen (within this page (or overall?)) //
///////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
for(QRecord record : records)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
@ -333,8 +373,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
ukErrorSummary
.withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values"
+ (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ")
+ StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues)))
+ (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ")
+ StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues)))
.withSingularFutureMessage(" record will not be")
.withPluralFutureMessage(" records will not be")

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java
View File

@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java
View File

@ -23,7 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil
import java.util.Iterator;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
/*******************************************************************************

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java
View File

@ -28,7 +28,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java
View File

@ -27,7 +27,7 @@ import java.util.Iterator;
import java.util.Locale;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
/*******************************************************************************

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java
View File

@ -27,7 +27,7 @@ import java.io.InputStream;
import java.io.Serializable;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.dhatim.fastexcel.reader.ReadableWorkbook;
import org.dhatim.fastexcel.reader.Sheet;

67
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java
View File

@ -32,8 +32,9 @@ import java.util.function.Supplier;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -62,8 +63,10 @@ public class BulkInsertMapping implements Serializable
private Map<String, Serializable> fieldNameToDefaultValueMap = new HashMap<>();
private Map<String, Map<String, Serializable>> fieldNameToValueMapping = new HashMap<>();
private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>();
private List<String> mappedAssociations = new ArrayList<>();
private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>();
private Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping = new HashMap<>();
private List<String> mappedAssociations = new ArrayList<>();
private Memoization<Pair<String, String>, Boolean> shouldProcessFieldForTable = new Memoization<>();
@ -72,11 +75,11 @@ public class BulkInsertMapping implements Serializable
/***************************************************************************
**
***************************************************************************/
public enum Layout
public enum Layout implements PossibleValueEnum<String>
{
FLAT(FlatRowsToRecord::new),
TALL(TallRowsToRecord::new),
WIDE(WideRowsToRecord::new);
WIDE(WideRowsToRecordWithExplicitMapping::new);
/***************************************************************************
@ -95,6 +98,7 @@ public class BulkInsertMapping implements Serializable
}
/***************************************************************************
**
***************************************************************************/
@ -102,6 +106,28 @@ public class BulkInsertMapping implements Serializable
{
return (supplier.get());
}
/***************************************************************************
**
***************************************************************************/
@Override
public String getPossibleValueId()
{
return name();
}
/***************************************************************************
**
***************************************************************************/
@Override
public String getPossibleValueLabel()
{
return StringUtils.ucFirst(name().toLowerCase());
}
}
@ -500,4 +526,35 @@ public class BulkInsertMapping implements Serializable
return (this);
}
/*******************************************************************************
** Getter for wideLayoutMapping
*******************************************************************************/
public Map<String, BulkInsertWideLayoutMapping> getWideLayoutMapping()
{
return (this.wideLayoutMapping);
}
/*******************************************************************************
** Setter for wideLayoutMapping
*******************************************************************************/
public void setWideLayoutMapping(Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping)
{
this.wideLayoutMapping = wideLayoutMapping;
}
/*******************************************************************************
** Fluent setter for wideLayoutMapping
*******************************************************************************/
public BulkInsertMapping withWideLayoutMapping(Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping)
{
this.wideLayoutMapping = wideLayoutMapping;
return (this);
}
}

216
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java Normal file
View File

@ -0,0 +1,216 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertWideLayoutMapping
{
private List<ChildRecordMapping> childRecordMappings;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public BulkInsertWideLayoutMapping(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
}
/***************************************************************************
**
***************************************************************************/
public static class ChildRecordMapping
{
Map<String, String> fieldNameToHeaderNameMaps;
Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public ChildRecordMapping(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public ChildRecordMapping(Map<String, String> fieldNameToHeaderNameMaps, Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
}
/*******************************************************************************
** Getter for fieldNameToHeaderNameMaps
*******************************************************************************/
public Map<String, String> getFieldNameToHeaderNameMaps()
{
return (this.fieldNameToHeaderNameMaps);
}
/*******************************************************************************
** Setter for fieldNameToHeaderNameMaps
*******************************************************************************/
public void setFieldNameToHeaderNameMaps(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
}
/*******************************************************************************
** Fluent setter for fieldNameToHeaderNameMaps
*******************************************************************************/
public ChildRecordMapping withFieldNameToHeaderNameMaps(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
return (this);
}
/*******************************************************************************
** Getter for associationNameToChildRecordMappingMap
*******************************************************************************/
public Map<String, BulkInsertWideLayoutMapping> getAssociationNameToChildRecordMappingMap()
{
return (this.associationNameToChildRecordMappingMap);
}
/*******************************************************************************
** Setter for associationNameToChildRecordMappingMap
*******************************************************************************/
public void setAssociationNameToChildRecordMappingMap(Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
}
/*******************************************************************************
** Fluent setter for associationNameToChildRecordMappingMap
*******************************************************************************/
public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
return (this);
}
/***************************************************************************
**
***************************************************************************/
public Map<String, Integer> getFieldIndexes(BulkLoadFileRow headerRow)
{
// todo memoize or otherwise don't recompute
Map<String, Integer> rs = new HashMap<>();
////////////////////////////////////////////////////////
// for the current file, map header values to indexes //
////////////////////////////////////////////////////////
Map<String, Integer> headerToIndexMap = new HashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
headerToIndexMap.put(headerValue, i);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// loop over fields - finding what header name they are mapped to - then what index that header is at. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
for(Map.Entry<String, String> entry : fieldNameToHeaderNameMaps.entrySet())
{
String headerName = entry.getValue();
if(headerName != null)
{
Integer headerIndex = headerToIndexMap.get(headerName);
if(headerIndex != null)
{
rs.put(entry.getKey(), headerIndex);
}
}
}
return (rs);
}
}
/*******************************************************************************
** Getter for childRecordMappings
*******************************************************************************/
public List<ChildRecordMapping> getChildRecordMappings()
{
return (this.childRecordMappings);
}
/*******************************************************************************
** Setter for childRecordMappings
*******************************************************************************/
public void setChildRecordMappings(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
}
/*******************************************************************************
** Fluent setter for childRecordMappings
*******************************************************************************/
public BulkInsertWideLayoutMapping withChildRecordMappings(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
return (this);
}
}

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java
View File

@ -30,8 +30,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
/*******************************************************************************
@ -62,13 +62,13 @@ public class FlatRowsToRecord implements RowsToRecordInterface
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName()));
setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()));
}
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping);
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java
View File

@ -26,8 +26,10 @@ import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -44,26 +46,50 @@ public interface RowsToRecordInterface
/***************************************************************************
**
** returns true if value from row was used, else false.
***************************************************************************/
default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index)
default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index)
{
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName;
String fieldName = field.getName();
QFieldType type = field.getType();
boolean valueFromRowWasUsed = false;
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName;
Serializable value = null;
if(index != null && row != null)
{
value = row.getValueElseNull(index);
if(value != null && !"".equals(value))
{
valueFromRowWasUsed = true;
}
}
else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix))
{
value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix);
}
/* note - moving this to ValueMapper...
if(value != null)
{
try
{
value = ValueUtils.getValueAsFieldType(type, value);
}
catch(Exception e)
{
record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]"));
}
}
*/
if(value != null)
{
record.setValue(fieldName, value);
}
return (valueFromRowWasUsed);
}
}

27
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java
View File

@ -35,8 +35,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -75,7 +75,12 @@ public class TallRowsToRecord implements RowsToRecordInterface
{
BulkLoadFileRow row = fileToRowsInterface.next();
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName());
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName());
if(CollectionUtils.nullSafeIsEmpty(groupByIndexes))
{
groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null);
}
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
if(rowGroupByValues == null)
{
@ -126,13 +131,24 @@ public class TallRowsToRecord implements RowsToRecordInterface
rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord));
}
ValueMapper.valueMapping(rs, mapping);
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private List<Integer> groupByAllIndexesFromTable(BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow headerRow, String name) throws QException
{
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, name, headerRow);
return new ArrayList<>(fieldIndexes.values());
}
/***************************************************************************
**
***************************************************************************/
@ -148,7 +164,7 @@ public class TallRowsToRecord implements RowsToRecordInterface
BulkLoadFileRow row = rows.get(0);
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
}
/////////////////////////////
@ -230,7 +246,8 @@ public class TallRowsToRecord implements RowsToRecordInterface
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls);
if(CollectionUtils.nullSafeIsEmpty(groupByIndexes))
{
throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls));
groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, associationNameChainForRecursiveCalls);
// throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls));
}
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);

72
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java
View File

@ -25,9 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
@ -35,12 +45,16 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
*******************************************************************************/
public class ValueMapper
{
private static final QLogger LOG = QLogger.getLogger(ValueMapper.class);
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping)
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table) throws QException
{
valueMapping(records, mapping, null);
valueMapping(records, mapping, table, null);
}
@ -48,7 +62,7 @@ public class ValueMapper
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, String associationNameChain)
private static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException
{
if(CollectionUtils.nullSafeIsEmpty(records))
{
@ -58,20 +72,58 @@ public class ValueMapper
Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
for(QRecord record : records)
{
for(Map.Entry<String, Map<String, Serializable>> entry : mappingForTable.entrySet())
for(Map.Entry<String, Serializable> valueEntry : record.getValues().entrySet())
{
String fieldName = entry.getKey();
Map<String, Serializable> map = entry.getValue();
String value = record.getValueString(fieldName);
if(value != null && map.containsKey(value))
QFieldMetaData field = table.getField(valueEntry.getKey());
Serializable value = valueEntry.getValue();
///////////////////
// value mappin' //
///////////////////
if(mappingForTable.containsKey(field.getName()) && value != null)
{
record.setValue(fieldName, map.get(value));
Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value));
if(mappedValue != null)
{
value = mappedValue;
}
}
/////////////////////
// type convertin' //
/////////////////////
if(value != null)
{
QFieldType type = field.getType();
try
{
value = ValueUtils.getValueAsFieldType(type, value);
}
catch(Exception e)
{
record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]"));
}
}
record.setValue(field.getName(), value);
}
//////////////////////////////////////
// recursively process associations //
//////////////////////////////////////
for(Map.Entry<String, List<QRecord>> entry : record.getAssociatedRecords().entrySet())
{
valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey());
String associationName = entry.getKey();
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isPresent())
{
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName);
}
else
{
throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]");
}
}
}
}

260
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java Normal file
View File

@ -0,0 +1,260 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = new QRecord();
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()));
}
processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record);
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private void processAssociations(Map<String, BulkInsertWideLayoutMapping> mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException
{
for(Map.Entry<String, BulkInsertWideLayoutMapping> entry : CollectionUtils.nonNullMap(mappingMap).entrySet())
{
String associationName = entry.getKey();
BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue();
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName;
for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings())
{
QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow);
if(associatedRecord != null)
{
record.withAssociatedRecord(associationName, associatedRecord);
}
}
}
}
/***************************************************************************
**
***************************************************************************/
private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException
{
Map<String, Integer> fieldIndexes = childRecordMapping.getFieldIndexes(headerRow);
QRecord associatedRecord = new QRecord();
boolean usedAnyValuesFromRow = false;
for(QFieldMetaData field : table.getFields().values())
{
boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
usedAnyValuesFromRow |= valueFromRowWasUsed;
}
if(usedAnyValuesFromRow)
{
processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord);
return (associatedRecord);
}
else
{
return (null);
}
}
// /***************************************************************************
// **
// ***************************************************************************/
// private List<QRecord> processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException
// {
// List<QRecord> rs = new ArrayList<>();
// Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
// for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
// {
// if(entry.getKey().startsWith(associationName + "."))
// {
// String fieldName = entry.getKey().substring(associationName.length() + 1);
// //////////////////////////////////////////////////////////////////////////
// // make sure the name here is for this table - not a sub-table under it //
// //////////////////////////////////////////////////////////////////////////
// if(!fieldName.contains("."))
// {
// fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue());
// }
// }
// }
// /////////////////////////////////////////////////////////////////////
// // loop over the length of the record, building associated records //
// /////////////////////////////////////////////////////////////////////
// QRecord associatedRecord = new QRecord();
// Set<String> processedFieldNames = new HashSet<>();
// boolean gotAnyValues = false;
// int subStartIndex = -1;
// for(int i = startIndex; i < endIndex; i++)
// {
// String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
// {
// ///////////////////////////////////////////////
// // ok - this is a value for this association //
// ///////////////////////////////////////////////
// if(subStartIndex == -1)
// {
// subStartIndex = i;
// }
// String fieldName = entry.getKey();
// if(processedFieldNames.contains(fieldName))
// {
// /////////////////////////////////////////////////
// // this means we're starting a new sub-record! //
// /////////////////////////////////////////////////
// if(gotAnyValues)
// {
// addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
// processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i);
// rs.add(associatedRecord);
// }
// associatedRecord = new QRecord();
// processedFieldNames = new HashSet<>();
// gotAnyValues = false;
// subStartIndex = i + 1;
// }
// processedFieldNames.add(fieldName);
// Serializable value = row.getValueElseNull(i);
// if(value != null && !"".equals(value))
// {
// gotAnyValues = true;
// }
// setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i);
// }
// }
// }
// ////////////////////////
// // handle final value //
// ////////////////////////
// if(gotAnyValues)
// {
// addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
// processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex);
// rs.add(associatedRecord);
// }
// return (rs);
// }
/***************************************************************************
**
***************************************************************************/
private void addDefaultValuesToAssociatedRecord(Set<String> processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain)
{
for(QFieldMetaData field : table.getFields().values())
{
if(!processedFieldNames.contains(field.getName()))
{
setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null);
}
}
}
}

78
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java
View File

@ -37,8 +37,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -48,7 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class WideRowsToRecord implements RowsToRecordInterface
public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
@ -77,7 +77,7 @@ public class WideRowsToRecord implements RowsToRecordInterface
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName()));
setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()));
}
processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size());
@ -85,7 +85,7 @@ public class WideRowsToRecord implements RowsToRecordInterface
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping);
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
@ -199,7 +199,7 @@ public class WideRowsToRecord implements RowsToRecordInterface
gotAnyValues = true;
}
setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i);
setValueOrDefault(associatedRecord, table.getField(fieldName), associationName, mapping, row, i);
}
}
}
@ -228,77 +228,11 @@ public class WideRowsToRecord implements RowsToRecordInterface
{
if(!processedFieldNames.contains(field.getName()))
{
setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null);
setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null);
}
}
}
/***************************************************************************
**
***************************************************************************/
// private List<QRecord> processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException
// {
// List<QRecord> rs = new ArrayList<>();
// String associationNameChainForRecursiveCalls = associationName;
// Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
// for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
// {
// if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + "."))
// {
// fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue());
// }
// }
// Map<String, List<Integer>> indexes = new HashMap<>();
// for(int i = 0; i < headerRow.size(); i++)
// {
// String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
// {
// indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i);
// }
// }
// }
// int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0);
// //////////////////////////////////////////////////////
// // figure out how many sub-rows we'll be processing //
// //////////////////////////////////////////////////////
// for(int i = 0; i < maxIndex; i++)
// {
// QRecord associatedRecord = new QRecord();
// boolean gotAnyValues = false;
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// String fieldName = entry.getKey();
// if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i)
// {
// Integer index = indexes.get(fieldName).get(i);
// Serializable value = row.getValueElseNull(index);
// if(value != null && !"".equals(value))
// {
// gotAnyValues = true;
// }
// setValueOrDefault(associatedRecord, fieldName, mapping, row, index);
// }
// }
// if(gotAnyValues)
// {
// processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size());
// rs.add(associatedRecord);
// }
// }
// return (rs);
// }
/***************************************************************************

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java
View File

@ -19,7 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable;

133
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java Normal file
View File

@ -0,0 +1,133 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable;
import java.util.ArrayList;
/***************************************************************************
* this is the model of a saved bulk load profile - which is what passes back
* and forth with the frontend.
****************************************************************************/
public class BulkLoadProfile implements Serializable
{
private ArrayList<BulkLoadProfileField> fieldList;
private Boolean hasHeaderRow;
private String layout;
/*******************************************************************************
** Getter for fieldList
*******************************************************************************/
public ArrayList<BulkLoadProfileField> getFieldList()
{
return (this.fieldList);
}
/*******************************************************************************
** Getter for hasHeaderRow
*******************************************************************************/
public Boolean getHasHeaderRow()
{
return (this.hasHeaderRow);
}
/*******************************************************************************
** Setter for hasHeaderRow
*******************************************************************************/
public void setHasHeaderRow(Boolean hasHeaderRow)
{
this.hasHeaderRow = hasHeaderRow;
}
/*******************************************************************************
** Fluent setter for hasHeaderRow
*******************************************************************************/
public BulkLoadProfile withHasHeaderRow(Boolean hasHeaderRow)
{
this.hasHeaderRow = hasHeaderRow;
return (this);
}
/*******************************************************************************
** Getter for layout
*******************************************************************************/
public String getLayout()
{
return (this.layout);
}
/*******************************************************************************
** Setter for layout
*******************************************************************************/
public void setLayout(String layout)
{
this.layout = layout;
}
/*******************************************************************************
** Fluent setter for layout
*******************************************************************************/
public BulkLoadProfile withLayout(String layout)
{
this.layout = layout;
return (this);
}
/*******************************************************************************
** Setter for fieldList
*******************************************************************************/
public void setFieldList(ArrayList<BulkLoadProfileField> fieldList)
{
this.fieldList = fieldList;
}
/*******************************************************************************
** Fluent setter for fieldList
*******************************************************************************/
public BulkLoadProfile withFieldList(ArrayList<BulkLoadProfileField> fieldList)
{
this.fieldList = fieldList;
return (this);
}
}

195
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java Normal file
View File

@ -0,0 +1,195 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable;
import java.util.Map;
/***************************************************************************
**
***************************************************************************/
public class BulkLoadProfileField
{
private String fieldName;
private Integer columnIndex;
private Serializable defaultValue;
private Boolean doValueMapping;
private Map<String, Serializable> valueMappings;
/*******************************************************************************
** Getter for fieldName
*******************************************************************************/
public String getFieldName()
{
return (this.fieldName);
}
/*******************************************************************************
** Setter for fieldName
*******************************************************************************/
public void setFieldName(String fieldName)
{
this.fieldName = fieldName;
}
/*******************************************************************************
** Fluent setter for fieldName
*******************************************************************************/
public BulkLoadProfileField withFieldName(String fieldName)
{
this.fieldName = fieldName;
return (this);
}
/*******************************************************************************
** Getter for columnIndex
*******************************************************************************/
public Integer getColumnIndex()
{
return (this.columnIndex);
}
/*******************************************************************************
** Setter for columnIndex
*******************************************************************************/
public void setColumnIndex(Integer columnIndex)
{
this.columnIndex = columnIndex;
}
/*******************************************************************************
** Fluent setter for columnIndex
*******************************************************************************/
public BulkLoadProfileField withColumnIndex(Integer columnIndex)
{
this.columnIndex = columnIndex;
return (this);
}
/*******************************************************************************
** Getter for defaultValue
*******************************************************************************/
public Serializable getDefaultValue()
{
return (this.defaultValue);
}
/*******************************************************************************
** Setter for defaultValue
*******************************************************************************/
public void setDefaultValue(Serializable defaultValue)
{
this.defaultValue = defaultValue;
}
/*******************************************************************************
** Fluent setter for defaultValue
*******************************************************************************/
public BulkLoadProfileField withDefaultValue(Serializable defaultValue)
{
this.defaultValue = defaultValue;
return (this);
}
/*******************************************************************************
** Getter for doValueMapping
*******************************************************************************/
public Boolean getDoValueMapping()
{
return (this.doValueMapping);
}
/*******************************************************************************
** Setter for doValueMapping
*******************************************************************************/
public void setDoValueMapping(Boolean doValueMapping)
{
this.doValueMapping = doValueMapping;
}
/*******************************************************************************
** Fluent setter for doValueMapping
*******************************************************************************/
public BulkLoadProfileField withDoValueMapping(Boolean doValueMapping)
{
this.doValueMapping = doValueMapping;
return (this);
}
/*******************************************************************************
** Getter for valueMappings
*******************************************************************************/
public Map<String, Serializable> getValueMappings()
{
return (this.valueMappings);
}
/*******************************************************************************
** Setter for valueMappings
*******************************************************************************/
public void setValueMappings(Map<String, Serializable> valueMappings)
{
this.valueMappings = valueMappings;
}
/*******************************************************************************
** Fluent setter for valueMappings
*******************************************************************************/
public BulkLoadProfileField withValueMappings(Map<String, Serializable> valueMappings)
{
this.valueMappings = valueMappings;
return (this);
}
}

275
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java Normal file
View File

@ -0,0 +1,275 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
/*******************************************************************************
**
*******************************************************************************/
public class BulkLoadTableStructure implements Serializable
{
private boolean isMain;
private boolean isMany;
private String tableName;
private String label;
private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild
private ArrayList<QFieldMetaData> fields; // mmm, not marked as serializable (at this time) - is okay?
private ArrayList<BulkLoadTableStructure> associations;
/*******************************************************************************
** Getter for isMain
*******************************************************************************/
public boolean getIsMain()
{
return (this.isMain);
}
/*******************************************************************************
** Setter for isMain
*******************************************************************************/
public void setIsMain(boolean isMain)
{
this.isMain = isMain;
}
/*******************************************************************************
** Fluent setter for isMain
*******************************************************************************/
public BulkLoadTableStructure withIsMain(boolean isMain)
{
this.isMain = isMain;
return (this);
}
/*******************************************************************************
** Getter for isMany
*******************************************************************************/
public boolean getIsMany()
{
return (this.isMany);
}
/*******************************************************************************
** Setter for isMany
*******************************************************************************/
public void setIsMany(boolean isMany)
{
this.isMany = isMany;
}
/*******************************************************************************
** Fluent setter for isMany
*******************************************************************************/
public BulkLoadTableStructure withIsMany(boolean isMany)
{
this.isMany = isMany;
return (this);
}
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getTableName()
{
return (this.tableName);
}
/*******************************************************************************
** Setter for tableName
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent setter for tableName
*******************************************************************************/
public BulkLoadTableStructure withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
/*******************************************************************************
** Getter for label
*******************************************************************************/
public String getLabel()
{
return (this.label);
}
/*******************************************************************************
** Setter for label
*******************************************************************************/
public void setLabel(String label)
{
this.label = label;
}
/*******************************************************************************
** Fluent setter for label
*******************************************************************************/
public BulkLoadTableStructure withLabel(String label)
{
this.label = label;
return (this);
}
/*******************************************************************************
** Getter for fields
*******************************************************************************/
public ArrayList<QFieldMetaData> getFields()
{
return (this.fields);
}
/*******************************************************************************
** Setter for fields
*******************************************************************************/
public void setFields(ArrayList<QFieldMetaData> fields)
{
this.fields = fields;
}
/*******************************************************************************
** Fluent setter for fields
*******************************************************************************/
public BulkLoadTableStructure withFields(ArrayList<QFieldMetaData> fields)
{
this.fields = fields;
return (this);
}
/*******************************************************************************
** Getter for associationPath
*******************************************************************************/
public String getAssociationPath()
{
return (this.associationPath);
}
/*******************************************************************************
** Setter for associationPath
*******************************************************************************/
public void setAssociationPath(String associationPath)
{
this.associationPath = associationPath;
}
/*******************************************************************************
** Fluent setter for associationPath
*******************************************************************************/
public BulkLoadTableStructure withAssociationPath(String associationPath)
{
this.associationPath = associationPath;
return (this);
}
/*******************************************************************************
** Getter for associations
*******************************************************************************/
public ArrayList<BulkLoadTableStructure> getAssociations()
{
return (this.associations);
}
/*******************************************************************************
** Setter for associations
*******************************************************************************/
public void setAssociations(ArrayList<BulkLoadTableStructure> associations)
{
this.associations = associations;
}
/*******************************************************************************
** Fluent setter for associations
*******************************************************************************/
public BulkLoadTableStructure withAssociations(ArrayList<BulkLoadTableStructure> associations)
{
this.associations = associations;
return (this);
}
/***************************************************************************
**
***************************************************************************/
public void addAssociation(BulkLoadTableStructure association)
{
if(this.associations == null)
{
this.associations = new ArrayList<>();
}
this.associations.add(association);
}
}

69
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java Normal file
View File

@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import com.kingsrook.qqq.backend.core.BaseTest;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for BulkInsertPrepareMappingStep
*******************************************************************************/
class BulkInsertPrepareFileMappingStepTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("PointlessArithmeticExpression")
@Test
void testToHeaderLetter()
{
assertEquals("A", BulkInsertPrepareFileMappingStep.toHeaderLetter(0));
assertEquals("B", BulkInsertPrepareFileMappingStep.toHeaderLetter(1));
assertEquals("Z", BulkInsertPrepareFileMappingStep.toHeaderLetter(25));
assertEquals("AA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 0));
assertEquals("AB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 1));
assertEquals("AZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 25));
assertEquals("BA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 0));
assertEquals("BB", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 1));
assertEquals("BZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 25));
assertEquals("ZA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 0));
assertEquals("ZB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 1));
assertEquals("ZZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 25));
assertEquals("AAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 0));
assertEquals("AAB", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 1));
assertEquals("AAC", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 2));
assertEquals("ABA", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 0));
assertEquals("ABB", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 1));
assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0));
}
}

42
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java → qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java
View File

@ -22,38 +22,34 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
**
** Unit test for BulkInsertPrepareValueMappingStep
*******************************************************************************/
public class BulkInsertReceiveMappingStep implements BackendStep
class BulkInsertPrepareValueMappingStepTest extends BaseTest
{
/***************************************************************************
/*******************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
*******************************************************************************/
@Test
void test() throws QException
{
BulkInsertMapping bulkInsertMapping = new BulkInsertMapping();
bulkInsertMapping.setTableName(runBackendStepInput.getTableName());
bulkInsertMapping.setHasHeaderRow(true);
bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of(
"firstName", "firstName",
"lastName", "Last Name"
));
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
assertEquals(TestUtils.TABLE_NAME_ORDER, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").table().getName());
assertEquals("orderNo", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").field().getName());
assertEquals(TestUtils.TABLE_NAME_LINE_ITEM, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").table().getName());
assertEquals("sku", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").field().getName());
assertEquals(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").table().getName());
assertEquals("key", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").field().getName());
// probably need to what, receive the mapping object, store it into state
// what, do we maybe return to a different sub-mapping screen (e.g., values)
// then at some point - cool - proceed to ETL's steps
}
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java
View File

@ -25,7 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil
import java.io.ByteArrayInputStream;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java
View File

@ -26,7 +26,7 @@ import java.io.InputStream;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
/***************************************************************************

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java
View File

@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java
View File

@ -29,8 +29,8 @@ import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
import org.junit.jupiter.api.Test;

101
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java
View File

@ -28,8 +28,8 @@ import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@ -87,14 +87,14 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
}
@ -145,7 +145,7 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(2, order.getAssociatedRecords().get("extrinsics").size());
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
@ -155,7 +155,7 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
}
@ -168,7 +168,7 @@ class TallRowsToRecordTest extends BaseTest
void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException
{
Integer defaultStoreId = 101;
Integer defaultLineNo = 102;
String defaultLineNo = "102";
String defaultOrderLineExtraSource = "file";
CsvFileToRows fileToRows = CsvFileToRows.forString("""
@ -221,7 +221,7 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
@ -240,7 +240,92 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAutomaticGroupByAllIndexes() throws QException
{
Integer defaultStoreId = 101;
String defaultLineNo = "102";
String defaultOrderLineExtraSource = "file";
CsvFileToRows fileToRows = CsvFileToRows.forString("""
orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value
1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate
1, Homer, Simpson, DONUT, 12, Coupon Code, 10QOff, Size, Large
1, Homer, Simpson, BEER, 500, , , Flavor, Hops
1, Homer, Simpson, COUCH, 1
2, Ned, Flanders, BIBLE, 7, , , Flavor, King James
2, Ned, Flanders, LAWNMOWER, 1
""");
BulkLoadFileRow header = fileToRows.next();
TallRowsToRecord rowsToRecord = new TallRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku", "SKU",
"orderLine.quantity", "Quantity",
"extrinsics.key", "Extrinsic Key",
"extrinsics.value", "Extrinsic Value",
"orderLine.extrinsics.key", "Line Extrinsic Key",
"orderLine.extrinsics.value", "Line Extrinsic Value"
))
.withFieldNameToDefaultValueMap(Map.of(
"storeId", defaultStoreId,
"orderLine.lineNumber", defaultLineNo,
"orderLine.extrinsics.source", defaultOrderLineExtraSource
))
.withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT")))
.withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.TALL)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
lineItem = order.getAssociatedRecords().get("orderLine").get(1);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();

7
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java
View File

@ -26,8 +26,11 @@ import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;
@ -44,7 +47,7 @@ class ValueMapperTest extends BaseTest
**
*******************************************************************************/
@Test
void test()
void test() throws QException
{
BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of(
"storeId", Map.of("QQQMart", 1, "Q'R'Us", 2),
@ -94,7 +97,7 @@ class ValueMapperTest extends BaseTest
);
JSONObject expectedJson = recordToJson(expectedRecord);
ValueMapper.valueMapping(List.of(inputRecord), mapping);
ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER));
JSONObject actualJson = recordToJson(inputRecord);
System.out.println("Before");

269
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java Normal file
View File

@ -0,0 +1,269 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for WideRowsToRecord
*******************************************************************************/
class WideRowsToRecordWithExplicitMappingTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderAndLinesWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withWideLayoutMapping(Map.of(
"orderLine", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3"))
))
))
.withMappedAssociations(List.of("orderLine"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withWideLayoutMapping(Map.of(
"orderLine", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3"))
)),
"extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2"))
))
))
.withMappedAssociations(List.of("orderLine", "extrinsics"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2
1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo,
2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1
""";
Integer defaultStoreId = 42;
Integer defaultLineNo = 47;
String defaultLineExtraValue = "bar";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
.withWideLayoutMapping(Map.of(
"orderLine", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(
Map.of("sku", "SKU 1", "quantity", "Quantity 1"),
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2"))
)))),
new BulkInsertWideLayoutMapping.ChildRecordMapping(
Map.of("sku", "SKU 2", "quantity", "Quantity 2"),
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2"))
)))),
new BulkInsertWideLayoutMapping.ChildRecordMapping(
Map.of("sku", "SKU 3", "quantity", "Quantity 3"),
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2"))
))))
)),
"extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2"))
))
))
.withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL")))
.withFieldNameToDefaultValueMap(Map.of(
"storeId", defaultStoreId,
"orderLine.lineNumber", defaultLineNo,
"orderLine.extrinsics.value", defaultLineExtraValue
))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
lineItem = order.getAssociatedRecords().get("orderLine").get(1);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
lineItem = order.getAssociatedRecords().get("orderLine").get(2);
assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
}
/***************************************************************************
**
***************************************************************************/
private List<Serializable> getValues(List<QRecord> records, String fieldName)
{
return (records.stream().map(r -> r.getValue(fieldName)).toList());
}
}

10
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java → qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java
View File

@ -28,8 +28,8 @@ import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@ -39,7 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for WideRowsToRecord
*******************************************************************************/
class WideRowsToRecordTest extends BaseTest
class WideRowsToRecordWithSpreadMappingTest extends BaseTest
{
/*******************************************************************************
@ -80,7 +80,7 @@ class WideRowsToRecordTest extends BaseTest
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecord rowsToRecord = new WideRowsToRecord();
WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
@ -150,7 +150,7 @@ class WideRowsToRecordTest extends BaseTest
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecord rowsToRecord = new WideRowsToRecord();
WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
@ -229,7 +229,7 @@ class WideRowsToRecordTest extends BaseTest
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecord rowsToRecord = new WideRowsToRecord();
WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(

4
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -644,6 +644,7 @@ public class TestUtils
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("orderNo", QFieldType.STRING))
.withField(new QFieldMetaData("shipToName", QFieldType.STRING))
.withField(new QFieldMetaData("orderDate", QFieldType.DATE))
.withField(new QFieldMetaData("storeId", QFieldType.INTEGER))
.withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock()
@ -700,7 +701,8 @@ public class TestUtils
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("lineItemId", QFieldType.INTEGER))
.withField(new QFieldMetaData("key", QFieldType.STRING))
.withField(new QFieldMetaData("value", QFieldType.STRING));
.withField(new QFieldMetaData("value", QFieldType.STRING))
.withField(new QFieldMetaData("source", QFieldType.STRING)); // doesn't really make sense, but useful to have an extra field here in some bulk-load tests
}