QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-20 06:00:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 Checkpoint on bulk-load backend

Browse Source
This commit is contained in:
darin.kelkhoff
2024-11-19 08:44:43 -06:00
parent b684f2409b
commit d8ac14a756
35 changed files with 2682 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java Normal file
View File

@ -0,0 +1,240 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertPrepareFileMappingStep implements BackendStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput);
buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput);
}
/***************************************************************************
**
***************************************************************************/
private static void buildFileDetailsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
File file = new File(storageInput.getReference());
runBackendStepOutput.addValue("fileBaseName", file.getName());
try
(
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// open a stream to read from our file, and a FileToRows object, that knows how to read from that stream //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
/////////////////////////////////////////////////
// read the 1st row, and assume it is a header //
/////////////////////////////////////////////////
BulkLoadFileRow headerRow = fileToRowsInterface.next();
ArrayList<String> headerValues = new ArrayList<>();
ArrayList<String> headerLetters = new ArrayList<>();
for(int i = 0; i < headerRow.size(); i++)
{
headerValues.add(ValueUtils.getValueAsString(headerRow.getValue(i)));
headerLetters.add(toHeaderLetter(i));
}
runBackendStepOutput.addValue("headerValues", headerValues);
runBackendStepOutput.addValue("headerLetters", headerLetters);
///////////////////////////////////////////////////////////////////////////////////////////
// while there are more rows in the file - and we're under preview-rows limit, read rows //
///////////////////////////////////////////////////////////////////////////////////////////
int previewRows = 0;
int previewRowsLimit = 5;
ArrayList<ArrayList<String>> bodyValues = new ArrayList<>();
for(int i = 0; i < headerRow.size(); i++)
{
bodyValues.add(new ArrayList<>());
}
while(fileToRowsInterface.hasNext() && previewRows < previewRowsLimit)
{
BulkLoadFileRow bodyRow = fileToRowsInterface.next();
previewRows++;
for(int i = 0; i < headerRow.size(); i++)
{
bodyValues.get(i).add(ValueUtils.getValueAsString(bodyRow.getValueElseNull(i)));
}
}
runBackendStepOutput.addValue("bodyValuesPreview", bodyValues);
}
catch(Exception e)
{
throw (new QException("Error reading bulk load file", e));
}
}
/***************************************************************************
**
***************************************************************************/
static String toHeaderLetter(int i)
{
StringBuilder rs = new StringBuilder();
do
{
rs.insert(0, (char) ('A' + (i % 26)));
i = (i / 26) - 1;
}
while(i >= 0);
return (rs.toString());
}
/***************************************************************************
**
***************************************************************************/
private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
String tableName = runBackendStepInput.getValueString("tableName");
BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null);
runBackendStepOutput.addValue("tableStructure", tableStructure);
}
/***************************************************************************
**
***************************************************************************/
private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath)
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
BulkLoadTableStructure tableStructure = new BulkLoadTableStructure();
tableStructure.setTableName(tableName);
tableStructure.setLabel(table.getLabel());
Set<String> associationJoinFieldNamesToExclude = new HashSet<>();
if(association == null)
{
tableStructure.setIsMain(true);
tableStructure.setIsMany(false);
tableStructure.setAssociationPath(null);
}
else
{
tableStructure.setIsMain(false);
QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName());
if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE))
{
tableStructure.setIsMany(true);
}
for(JoinOn joinOn : join.getJoinOns())
{
////////////////////////////////////////////////////////////////////////////////////////////////
// don't allow the user to map the "join field" from a child up to its parent //
// (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) //
////////////////////////////////////////////////////////////////////////////////////////////////
if(join.getLeftTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getLeftField());
}
else if(join.getRightTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getRightField());
}
}
if(!StringUtils.hasContent(parentAssociationPath))
{
tableStructure.setAssociationPath(association.getName());
}
else
{
tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName());
}
}
ArrayList<QFieldMetaData> fields = new ArrayList<>();
tableStructure.setFields(fields);
for(QFieldMetaData field : table.getFields().values())
{
if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName()))
{
fields.add(field);
}
}
fields.sort(Comparator.comparing(f -> f.getLabel()));
for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations()))
{
BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath);
tableStructure.addAssociation(associatedStructure);
}
return (tableStructure);
}
}

250
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java Normal file
View File

@ -0,0 +1,250 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.google.gson.reflect.TypeToken;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertPrepareValueMappingStep implements BackendStep
{
private static final QLogger LOG = QLogger.getLogger(BulkInsertPrepareValueMappingStep.class);
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
try
{
/////////////////////////////////////////////////////////////
// prep the frontend for what field we're going to map now //
/////////////////////////////////////////////////////////////
List<String> fieldNamesToDoValueMapping = (List<String>) runBackendStepInput.getValue("fieldNamesToDoValueMapping");
Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex");
if(valueMappingFieldIndex == null)
{
valueMappingFieldIndex = 0;
}
else
{
valueMappingFieldIndex++;
}
////////////////////////////////////////////////////////////////////////////////////////////////////
// if there are no more fields (values) to map, then proceed to the standard streamed-ETL preview //
////////////////////////////////////////////////////////////////////////////////////////////////////
if(valueMappingFieldIndex >= fieldNamesToDoValueMapping.size())
{
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
return;
}
runBackendStepInput.addValue("valueMappingFieldIndex", valueMappingFieldIndex);
String fullFieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex);
TableAndField tableAndField = getTableAndField(runBackendStepInput.getValueString("tableName"), fullFieldName);
runBackendStepInput.addValue("valueMappingField", new QFrontendFieldMetaData(tableAndField.field()));
runBackendStepInput.addValue("valueMappingFullFieldName", fullFieldName);
runBackendStepInput.addValue("valueMappingFieldTableName", tableAndField.table().getName());
////////////////////////////////////////////////////
// get all the values from the file in this field //
// todo - should do all mapping fields at once? //
////////////////////////////////////////////////////
ArrayList<Serializable> fileValues = getValuesForField(tableAndField.table(), tableAndField.field(), fullFieldName, runBackendStepInput);
runBackendStepOutput.addValue("fileValues", fileValues);
///////////////////////////////////////////////
// clear these in case not getting set below //
///////////////////////////////////////////////
runBackendStepOutput.addValue("valueMapping", new HashMap<>());
runBackendStepOutput.addValue("mappedValueLabels", new HashMap<>());
BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping");
HashMap<String, Serializable> valueMapping = null;
if(bulkInsertMapping.getFieldNameToValueMapping() != null && bulkInsertMapping.getFieldNameToValueMapping().containsKey(fullFieldName))
{
valueMapping = CollectionUtils.useOrWrap(bulkInsertMapping.getFieldNameToValueMapping().get(fullFieldName), new TypeToken<>() {});
runBackendStepOutput.addValue("valueMapping", valueMapping);
if(StringUtils.hasContent(tableAndField.field().getPossibleValueSourceName()))
{
HashMap<Serializable, String> possibleValueLabels = loadPossibleValues(tableAndField.field(), valueMapping);
runBackendStepOutput.addValue("mappedValueLabels", possibleValueLabels);
}
}
}
catch(Exception e)
{
LOG.warn("Error in bulk insert prepare value mapping", e);
throw new QException("Unhandled error in bulk insert prepare value mapping step", e);
}
}
/***************************************************************************
**
***************************************************************************/
public static TableAndField getTableAndField(String tableName, String fullFieldName) throws QException
{
List<String> parts = new ArrayList<>(List.of(fullFieldName.split("\\.")));
String fieldBaseName = parts.remove(parts.size() - 1);
QTableMetaData table = QContext.getQInstance().getTable(tableName);
for(String associationName : parts)
{
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isPresent())
{
table = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
}
else
{
throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]");
}
}
TableAndField result = new TableAndField(table, table.getField(fieldBaseName));
return result;
}
/***************************************************************************
**
***************************************************************************/
public record TableAndField(QTableMetaData table, QFieldMetaData field) {}
/***************************************************************************
**
***************************************************************************/
private HashMap<Serializable, String> loadPossibleValues(QFieldMetaData field, Map<String, Serializable> valueMapping) throws QException
{
SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput();
input.setPossibleValueSourceName(field.getPossibleValueSourceName());
input.setIdList(new ArrayList<>(new HashSet<>(valueMapping.values()))); // go through a set to strip dupes
SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input);
HashMap<Serializable, String> rs = new HashMap<>();
for(QPossibleValue<?> result : output.getResults())
{
Serializable id = (Serializable) result.getId();
rs.put(id, result.getLabel());
}
return rs;
}
/***************************************************************************
**
***************************************************************************/
private ArrayList<Serializable> getValuesForField(QTableMetaData table, QFieldMetaData field, String fullFieldName, RunBackendStepInput runBackendStepInput) throws QException
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping");
String associationNameChain = null;
if(fullFieldName.contains("."))
{
associationNameChain = fullFieldName.substring(0, fullFieldName.lastIndexOf('.'));
}
try
(
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
Set<String> values = new LinkedHashSet<>();
BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null;
Map<String, Integer> fieldIndexes = bulkInsertMapping.getFieldIndexes(table, associationNameChain, headerRow);
int index = fieldIndexes.get(field.getName());
while(fileToRowsInterface.hasNext())
{
BulkLoadFileRow row = fileToRowsInterface.next();
Serializable value = row.getValueElseNull(index);
if(value != null)
{
values.add(ValueUtils.getValueAsString(value));
}
if(values.size() > 100)
{
throw (new QUserFacingException("Too many unique values were found for mapping for field: " + field.getName()));
}
}
return (new ArrayList<>(values));
}
catch(Exception e)
{
throw (new QException("Error getting values from file", e));
}
}
}

200
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java Normal file
View File

@ -0,0 +1,200 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveFileMappingStep implements BackendStep
{
private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveFileMappingStep.class);
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
try
{
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
/////////////////////////////////////////////////////////////////////////
// put the list of bulk load profile into the process state - it's the //
// thing that the frontend will be looking at as the saved profile //
/////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile);
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// now build the mapping object that the backend wants - based on the bulkLoadProfile from the frontend //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertMapping bulkInsertMapping = new BulkInsertMapping();
bulkInsertMapping.setTableName(runBackendStepInput.getTableName());
bulkInsertMapping.setHasHeaderRow(bulkLoadProfile.getHasHeaderRow());
bulkInsertMapping.setLayout(BulkInsertMapping.Layout.valueOf(bulkLoadProfile.getLayout()));
//////////////////////////////////////////////////////////////////////////////////////////////
// handle field to name or index mappings (depending on if there's a header row being used) //
//////////////////////////////////////////////////////////////////////////////////////////////
if(BooleanUtils.isTrue(bulkInsertMapping.getHasHeaderRow()))
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
try
(
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
Map<String, String> fieldNameToHeaderNameMap = new HashMap<>();
bulkInsertMapping.setFieldNameToHeaderNameMap(fieldNameToHeaderNameMap);
BulkLoadFileRow headerRow = fileToRowsInterface.next();
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getColumnIndex() != null)
{
String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex()));
fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName);
}
}
}
}
else
{
Map<String, Integer> fieldNameToIndexMap = new HashMap<>();
bulkInsertMapping.setFieldNameToIndexMap(fieldNameToIndexMap);
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getColumnIndex() != null)
{
fieldNameToIndexMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getColumnIndex());
}
}
}
/////////////////////////////////////
// do fields w/ default values now //
/////////////////////////////////////
HashMap<String, Serializable> fieldNameToDefaultValueMap = new HashMap<>();
bulkInsertMapping.setFieldNameToDefaultValueMap(fieldNameToDefaultValueMap);
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getDefaultValue() != null)
{
fieldNameToDefaultValueMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getDefaultValue());
}
}
/////////////////////////////////////////////////////////////////////////////////////////////
// frontend at this point will have sent just told us which field names need value mapping //
// store those - and let them drive the value-mapping screens that we'll go through next //
// todo - uh, what if those come from profile, dummy!?
/////////////////////////////////////////////////////////////////////////////////////////////
ArrayList<String> fieldNamesToDoValueMapping = new ArrayList<>();
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()))
{
fieldNamesToDoValueMapping.add(bulkLoadProfileField.getFieldName());
if(CollectionUtils.nullSafeHasContents(bulkLoadProfileField.getValueMappings()))
{
bulkInsertMapping.getFieldNameToValueMapping().put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getValueMappings());
}
}
}
runBackendStepOutput.addValue("fieldNamesToDoValueMapping", new ArrayList<>(fieldNamesToDoValueMapping));
///////////////////////////////////////////////////////////////////////////////////////
// figure out what associations are being mapped, by looking at the full field names //
///////////////////////////////////////////////////////////////////////////////////////
Set<String> associationNameSet = new HashSet<>();
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getFieldName().contains("."))
{
associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.')));
}
}
bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet));
/////////////////////////////////////////////////////////////////////////////////////////////////////
// at this point we're done populating the bulkInsertMapping object. put it in the process state. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping))
{
//////////////////////////////////////////////////////////////////////////////////
// just go to the prepareValueMapping backend step - it'll figure out the rest. //
// it's also where the value-mapping loop of steps points. //
// and, this will actually be the default (e.g., the step after this one). //
//////////////////////////////////////////////////////////////////////////////////
}
else
{
//////////////////////////////////////////////////////////////////////////////////
// else - if no values to map - continue with the standard streamed-ETL preview //
//////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
}
}
catch(Exception e)
{
LOG.warn("Error in bulk insert receive mapping", e);
throw new QException("Unhandled error in bulk insert receive mapping step", e);
}
}
}

81
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java
View File

@ -1,81 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveFileStep implements BackendStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
try
(
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
BulkLoadFileRow headerRow = fileToRowsInterface.next();
List<String> bodyRows = new ArrayList<>();
while(fileToRowsInterface.hasNext() && bodyRows.size() < 20)
{
bodyRows.add(fileToRowsInterface.next().toString());
}
runBackendStepOutput.addValue("header", headerRow.toString());
runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows));
System.out.println("Done receiving file");
}
catch(QException qe)
{
throw qe;
}
catch(Exception e)
{
throw new QException("Unhandled error in bulk insert extract step", e);
}
}
}

59
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java
View File

@ -1,59 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveMappingStep implements BackendStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
BulkInsertMapping bulkInsertMapping = new BulkInsertMapping();
bulkInsertMapping.setTableName(runBackendStepInput.getTableName());
bulkInsertMapping.setHasHeaderRow(true);
bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of(
"firstName", "firstName",
"lastName", "Last Name"
));
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
// probably need to what, receive the mapping object, store it into state
// what, do we maybe return to a different sub-mapping screen (e.g., values)
// then at some point - cool - proceed to ETL's steps
}
}

105
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java Normal file
View File

@ -0,0 +1,105 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.core.type.TypeReference;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveValueMappingStep implements BackendStep
{
private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveValueMappingStep.class);
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
try
{
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
List<String> fieldNamesToDoValueMapping = (List<String>) runBackendStepInput.getValue("fieldNamesToDoValueMapping");
Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex");
String fieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex);
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
/////////////////////////////////////////////////////////////////////////
// put the list of bulk load profile into the process state - it's the //
// thing that the frontend will be looking at as the saved profile //
/////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// get the bulkInsertMapping object from the process, creating a fieldNameToValueMapping map within it if needed //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping");
Map<String, Map<String, Serializable>> fieldNameToValueMapping = bulkInsertMapping.getFieldNameToValueMapping();
if(fieldNameToValueMapping == null)
{
fieldNameToValueMapping = new HashMap<>();
bulkInsertMapping.setFieldNameToValueMapping(fieldNameToValueMapping);
}
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
////////////////////////////////////////////////
// put the mapped values into the mapping map //
////////////////////////////////////////////////
Map<String, Serializable> mappedValues = JsonUtils.toObject(runBackendStepInput.getValueString("mappedValuesJSON"), new TypeReference<>() {});
fieldNameToValueMapping.put(fieldName, mappedValues);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
// always return to the prepare-mapping step - as it will determine if it's time to break the loop or not. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepPrepareValueMapping(runBackendStepOutput);
}
catch(Exception e)
{
LOG.warn("Error in bulk insert receive mapping", e);
throw new QException("Unhandled error in bulk insert receive mapping step", e);
}
}
}

96
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java
View File

@ -22,10 +22,22 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.json.JSONArray;
import org.json.JSONObject;
/*******************************************************************************
@ -55,4 +67,88 @@ public class BulkInsertStepUtils
return (storageInput);
}
/***************************************************************************
**
***************************************************************************/
public static void setNextStepStreamedETLPreview(RunBackendStepOutput runBackendStepOutput)
{
runBackendStepOutput.setOverrideLastStepName("receiveValueMapping");
}
/***************************************************************************
**
***************************************************************************/
public static void setNextStepPrepareValueMapping(RunBackendStepOutput runBackendStepOutput)
{
runBackendStepOutput.setOverrideLastStepName("receiveFileMapping");
}
/***************************************************************************
**
***************************************************************************/
public static BulkLoadProfile getBulkLoadProfile(RunBackendStepInput runBackendStepInput)
{
String version = runBackendStepInput.getValueString("version");
if("v1".equals(version))
{
String layout = runBackendStepInput.getValueString("layout");
Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow");
ArrayList<BulkLoadProfileField> fieldList = new ArrayList<>();
JSONArray array = new JSONArray(runBackendStepInput.getValueString("fieldListJSON"));
for(int i = 0; i < array.length(); i++)
{
JSONObject jsonObject = array.getJSONObject(i);
BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField();
fieldList.add(bulkLoadProfileField);
bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName"));
bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null);
bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue"));
bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping"));
if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings"))
{
bulkLoadProfileField.setValueMappings(new HashMap<>());
JSONObject valueMappingsJsonObject = jsonObject.getJSONObject("valueMappings");
for(String fileValue : valueMappingsJsonObject.keySet())
{
bulkLoadProfileField.getValueMappings().put(fileValue, ValueUtils.getValueAsString(valueMappingsJsonObject.get(fileValue)));
}
}
}
BulkLoadProfile bulkLoadProfile = new BulkLoadProfile()
.withFieldList(fieldList)
.withHasHeaderRow(hasHeaderRow)
.withLayout(layout);
return (bulkLoadProfile);
}
else
{
throw (new IllegalArgumentException("Unexpected version for bulk load profile: " + version));
}
}
/***************************************************************************
**
***************************************************************************/
public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId");
if(savedBulkLoadProfileId != null)
{
QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId);
runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord);
}
}
}

68
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -111,6 +111,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep
// since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue(LoadViaInsertStep.FIELD_SKIP_UNIQUE_KEY_CHECK, true);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
// make sure that if a saved profile was selected on a review screen, that the result screen knows about it. //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
}
@ -121,8 +126,43 @@ public class BulkInsertTransformStep extends AbstractTransformStep
@Override
public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
int rowsInThisPage = runBackendStepInput.getRecords().size();
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
int recordsInThisPage = runBackendStepInput.getRecords().size();
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
// split the records w/o UK errors into those w/ e
List<QRecord> recordsWithoutAnyErrors = new ArrayList<>();
List<QRecord> recordsWithSomeErrors = new ArrayList<>();
for(QRecord record : runBackendStepInput.getRecords())
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
recordsWithSomeErrors.add(record);
}
else
{
recordsWithoutAnyErrors.add(record);
}
}
//////////////////////////////////////////////////////////////////
// propagate errors that came into this step out to the summary //
//////////////////////////////////////////////////////////////////
if(!recordsWithSomeErrors.isEmpty())
{
for(QRecord record : recordsWithSomeErrors)
{
String message = record.getErrors().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addError(message, null);
}
}
if(recordsWithoutAnyErrors.isEmpty())
{
////////////////////////////////////////////////////////////////////////////////
// skip th rest of this method if there aren't any records w/o errors in them //
////////////////////////////////////////////////////////////////////////////////
this.rowsProcessed += recordsInThisPage;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations //
@ -130,7 +170,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
InsertInput insertInput = new InsertInput();
insertInput.setInputSource(QInputSource.USER);
insertInput.setTableName(runBackendStepInput.getTableName());
insertInput.setRecords(runBackendStepInput.getRecords());
insertInput.setRecords(recordsWithoutAnyErrors);
insertInput.setSkipUniqueKeyCheck(true);
//////////////////////////////////////////////////////////////////////
@ -145,7 +185,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true);
if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun))
{
List<QRecord> recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, runBackendStepInput.getRecords(), true);
List<QRecord> recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, recordsWithoutAnyErrors, true);
runBackendStepInput.setRecords(recordsAfterCustomizer);
///////////////////////////////////////////////////////////////////////////////////////
@ -159,13 +199,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep
List<UniqueKey> uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys());
for(UniqueKey uniqueKey : uniqueKeys)
{
existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet());
existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, recordsWithoutAnyErrors, uniqueKey).keySet());
ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// on the validate step, we haven't read the full file, so we don't know how many rows there are - thus //
// record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
// todo - move this up (before the early return?) //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
{
@ -187,12 +228,12 @@ public class BulkInsertTransformStep extends AbstractTransformStep
// Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction //
// will only be getting the records in pages, but in here, we'll track UK's across pages!! //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table);
List<QRecord> recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(recordsWithoutAnyErrors, existingKeys, uniqueKeys, table);
/////////////////////////////////////////////////////////////////////////////////
// run all validation from the insert action - in Preview mode (boolean param) //
/////////////////////////////////////////////////////////////////////////////////
insertInput.setRecords(recordsWithoutUkErrors);
insertInput.setRecords(recordsWithoutAnyErrors);
InsertAction insertAction = new InsertAction();
insertAction.performValidations(insertInput, true);
List<QRecord> validationResultRecords = insertInput.getRecords();
@ -222,8 +263,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
}
runBackendStepOutput.setRecords(outputRecords);
this.rowsProcessed += rowsInThisPage;
this.rowsProcessed += recordsInThisPage;
}
@ -231,7 +271,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
/*******************************************************************************
**
*******************************************************************************/
private List<QRecord> getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map<UniqueKey, Set<List<Serializable>>> existingKeys, List<UniqueKey> uniqueKeys, QTableMetaData table)
private List<QRecord> getRecordsWithoutUniqueKeyErrors(List<QRecord> records, Map<UniqueKey, Set<List<Serializable>>> existingKeys, List<UniqueKey> uniqueKeys, QTableMetaData table)
{
////////////////////////////////////////////////////
// if there are no UK's, proceed with all records //
@ -239,7 +279,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
List<QRecord> recordsWithoutUkErrors = new ArrayList<>();
if(existingKeys.isEmpty())
{
recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords());
recordsWithoutUkErrors.addAll(records);
}
else
{
@ -255,7 +295,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
// else, get each records keys and see if it already exists or not //
// also, build a set of keys we've seen (within this page (or overall?)) //
///////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
for(QRecord record : records)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
@ -333,8 +373,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
ukErrorSummary
.withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values"
+ (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ")
+ StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues)))
+ (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ")
+ StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues)))
.withSingularFutureMessage(" record will not be")
.withPluralFutureMessage(" records will not be")

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java
View File

@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java
View File

@ -23,7 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil
import java.util.Iterator;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
/*******************************************************************************

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java
View File

@ -28,7 +28,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java
View File

@ -27,7 +27,7 @@ import java.util.Iterator;
import java.util.Locale;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
/*******************************************************************************

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java
View File

@ -27,7 +27,7 @@ import java.io.InputStream;
import java.io.Serializable;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.dhatim.fastexcel.reader.ReadableWorkbook;
import org.dhatim.fastexcel.reader.Sheet;

67
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java
View File

@ -32,8 +32,9 @@ import java.util.function.Supplier;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -62,8 +63,10 @@ public class BulkInsertMapping implements Serializable
private Map<String, Serializable> fieldNameToDefaultValueMap = new HashMap<>();
private Map<String, Map<String, Serializable>> fieldNameToValueMapping = new HashMap<>();
private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>();
private List<String> mappedAssociations = new ArrayList<>();
private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>();
private Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping = new HashMap<>();
private List<String> mappedAssociations = new ArrayList<>();
private Memoization<Pair<String, String>, Boolean> shouldProcessFieldForTable = new Memoization<>();
@ -72,11 +75,11 @@ public class BulkInsertMapping implements Serializable
/***************************************************************************
**
***************************************************************************/
public enum Layout
public enum Layout implements PossibleValueEnum<String>
{
FLAT(FlatRowsToRecord::new),
TALL(TallRowsToRecord::new),
WIDE(WideRowsToRecord::new);
WIDE(WideRowsToRecordWithExplicitMapping::new);
/***************************************************************************
@ -95,6 +98,7 @@ public class BulkInsertMapping implements Serializable
}
/***************************************************************************
**
***************************************************************************/
@ -102,6 +106,28 @@ public class BulkInsertMapping implements Serializable
{
return (supplier.get());
}
/***************************************************************************
**
***************************************************************************/
@Override
public String getPossibleValueId()
{
return name();
}
/***************************************************************************
**
***************************************************************************/
@Override
public String getPossibleValueLabel()
{
return StringUtils.ucFirst(name().toLowerCase());
}
}
@ -500,4 +526,35 @@ public class BulkInsertMapping implements Serializable
return (this);
}
/*******************************************************************************
** Getter for wideLayoutMapping
*******************************************************************************/
public Map<String, BulkInsertWideLayoutMapping> getWideLayoutMapping()
{
return (this.wideLayoutMapping);
}
/*******************************************************************************
** Setter for wideLayoutMapping
*******************************************************************************/
public void setWideLayoutMapping(Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping)
{
this.wideLayoutMapping = wideLayoutMapping;
}
/*******************************************************************************
** Fluent setter for wideLayoutMapping
*******************************************************************************/
public BulkInsertMapping withWideLayoutMapping(Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping)
{
this.wideLayoutMapping = wideLayoutMapping;
return (this);
}
}

216
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java Normal file
View File

@ -0,0 +1,216 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertWideLayoutMapping
{
private List<ChildRecordMapping> childRecordMappings;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public BulkInsertWideLayoutMapping(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
}
/***************************************************************************
**
***************************************************************************/
public static class ChildRecordMapping
{
Map<String, String> fieldNameToHeaderNameMaps;
Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public ChildRecordMapping(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public ChildRecordMapping(Map<String, String> fieldNameToHeaderNameMaps, Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
}
/*******************************************************************************
** Getter for fieldNameToHeaderNameMaps
*******************************************************************************/
public Map<String, String> getFieldNameToHeaderNameMaps()
{
return (this.fieldNameToHeaderNameMaps);
}
/*******************************************************************************
** Setter for fieldNameToHeaderNameMaps
*******************************************************************************/
public void setFieldNameToHeaderNameMaps(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
}
/*******************************************************************************
** Fluent setter for fieldNameToHeaderNameMaps
*******************************************************************************/
public ChildRecordMapping withFieldNameToHeaderNameMaps(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
return (this);
}
/*******************************************************************************
** Getter for associationNameToChildRecordMappingMap
*******************************************************************************/
public Map<String, BulkInsertWideLayoutMapping> getAssociationNameToChildRecordMappingMap()
{
return (this.associationNameToChildRecordMappingMap);
}
/*******************************************************************************
** Setter for associationNameToChildRecordMappingMap
*******************************************************************************/
public void setAssociationNameToChildRecordMappingMap(Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
}
/*******************************************************************************
** Fluent setter for associationNameToChildRecordMappingMap
*******************************************************************************/
public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
return (this);
}
/***************************************************************************
**
***************************************************************************/
public Map<String, Integer> getFieldIndexes(BulkLoadFileRow headerRow)
{
// todo memoize or otherwise don't recompute
Map<String, Integer> rs = new HashMap<>();
////////////////////////////////////////////////////////
// for the current file, map header values to indexes //
////////////////////////////////////////////////////////
Map<String, Integer> headerToIndexMap = new HashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
headerToIndexMap.put(headerValue, i);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// loop over fields - finding what header name they are mapped to - then what index that header is at. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
for(Map.Entry<String, String> entry : fieldNameToHeaderNameMaps.entrySet())
{
String headerName = entry.getValue();
if(headerName != null)
{
Integer headerIndex = headerToIndexMap.get(headerName);
if(headerIndex != null)
{
rs.put(entry.getKey(), headerIndex);
}
}
}
return (rs);
}
}
/*******************************************************************************
** Getter for childRecordMappings
*******************************************************************************/
public List<ChildRecordMapping> getChildRecordMappings()
{
return (this.childRecordMappings);
}
/*******************************************************************************
** Setter for childRecordMappings
*******************************************************************************/
public void setChildRecordMappings(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
}
/*******************************************************************************
** Fluent setter for childRecordMappings
*******************************************************************************/
public BulkInsertWideLayoutMapping withChildRecordMappings(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
return (this);
}
}

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java
View File

@ -30,8 +30,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
/*******************************************************************************
@ -62,13 +62,13 @@ public class FlatRowsToRecord implements RowsToRecordInterface
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName()));
setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()));
}
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping);
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java
View File

@ -26,8 +26,10 @@ import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -44,26 +46,50 @@ public interface RowsToRecordInterface
/***************************************************************************
**
** returns true if value from row was used, else false.
***************************************************************************/
default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index)
default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index)
{
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName;
String fieldName = field.getName();
QFieldType type = field.getType();
boolean valueFromRowWasUsed = false;
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName;
Serializable value = null;
if(index != null && row != null)
{
value = row.getValueElseNull(index);
if(value != null && !"".equals(value))
{
valueFromRowWasUsed = true;
}
}
else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix))
{
value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix);
}
/* note - moving this to ValueMapper...
if(value != null)
{
try
{
value = ValueUtils.getValueAsFieldType(type, value);
}
catch(Exception e)
{
record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]"));
}
}
*/
if(value != null)
{
record.setValue(fieldName, value);
}
return (valueFromRowWasUsed);
}
}

27
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java
View File

@ -35,8 +35,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -75,7 +75,12 @@ public class TallRowsToRecord implements RowsToRecordInterface
{
BulkLoadFileRow row = fileToRowsInterface.next();
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName());
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName());
if(CollectionUtils.nullSafeIsEmpty(groupByIndexes))
{
groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null);
}
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
if(rowGroupByValues == null)
{
@ -126,13 +131,24 @@ public class TallRowsToRecord implements RowsToRecordInterface
rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord));
}
ValueMapper.valueMapping(rs, mapping);
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private List<Integer> groupByAllIndexesFromTable(BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow headerRow, String name) throws QException
{
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, name, headerRow);
return new ArrayList<>(fieldIndexes.values());
}
/***************************************************************************
**
***************************************************************************/
@ -148,7 +164,7 @@ public class TallRowsToRecord implements RowsToRecordInterface
BulkLoadFileRow row = rows.get(0);
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
}
/////////////////////////////
@ -230,7 +246,8 @@ public class TallRowsToRecord implements RowsToRecordInterface
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls);
if(CollectionUtils.nullSafeIsEmpty(groupByIndexes))
{
throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls));
groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, associationNameChainForRecursiveCalls);
// throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls));
}
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);

72
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java
View File

@ -25,9 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
@ -35,12 +45,16 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
*******************************************************************************/
public class ValueMapper
{
private static final QLogger LOG = QLogger.getLogger(ValueMapper.class);
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping)
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table) throws QException
{
valueMapping(records, mapping, null);
valueMapping(records, mapping, table, null);
}
@ -48,7 +62,7 @@ public class ValueMapper
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, String associationNameChain)
private static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException
{
if(CollectionUtils.nullSafeIsEmpty(records))
{
@ -58,20 +72,58 @@ public class ValueMapper
Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
for(QRecord record : records)
{
for(Map.Entry<String, Map<String, Serializable>> entry : mappingForTable.entrySet())
for(Map.Entry<String, Serializable> valueEntry : record.getValues().entrySet())
{
String fieldName = entry.getKey();
Map<String, Serializable> map = entry.getValue();
String value = record.getValueString(fieldName);
if(value != null && map.containsKey(value))
QFieldMetaData field = table.getField(valueEntry.getKey());
Serializable value = valueEntry.getValue();
///////////////////
// value mappin' //
///////////////////
if(mappingForTable.containsKey(field.getName()) && value != null)
{
record.setValue(fieldName, map.get(value));
Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value));
if(mappedValue != null)
{
value = mappedValue;
}
}
/////////////////////
// type convertin' //
/////////////////////
if(value != null)
{
QFieldType type = field.getType();
try
{
value = ValueUtils.getValueAsFieldType(type, value);
}
catch(Exception e)
{
record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]"));
}
}
record.setValue(field.getName(), value);
}
//////////////////////////////////////
// recursively process associations //
//////////////////////////////////////
for(Map.Entry<String, List<QRecord>> entry : record.getAssociatedRecords().entrySet())
{
valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey());
String associationName = entry.getKey();
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isPresent())
{
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName);
}
else
{
throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]");
}
}
}
}

260
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java Normal file
View File

@ -0,0 +1,260 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = new QRecord();
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()));
}
processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record);
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private void processAssociations(Map<String, BulkInsertWideLayoutMapping> mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException
{
for(Map.Entry<String, BulkInsertWideLayoutMapping> entry : CollectionUtils.nonNullMap(mappingMap).entrySet())
{
String associationName = entry.getKey();
BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue();
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName;
for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings())
{
QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow);
if(associatedRecord != null)
{
record.withAssociatedRecord(associationName, associatedRecord);
}
}
}
}
/***************************************************************************
**
***************************************************************************/
private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException
{
Map<String, Integer> fieldIndexes = childRecordMapping.getFieldIndexes(headerRow);
QRecord associatedRecord = new QRecord();
boolean usedAnyValuesFromRow = false;
for(QFieldMetaData field : table.getFields().values())
{
boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
usedAnyValuesFromRow |= valueFromRowWasUsed;
}
if(usedAnyValuesFromRow)
{
processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord);
return (associatedRecord);
}
else
{
return (null);
}
}
// /***************************************************************************
// **
// ***************************************************************************/
// private List<QRecord> processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException
// {
// List<QRecord> rs = new ArrayList<>();
// Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
// for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
// {
// if(entry.getKey().startsWith(associationName + "."))
// {
// String fieldName = entry.getKey().substring(associationName.length() + 1);
// //////////////////////////////////////////////////////////////////////////
// // make sure the name here is for this table - not a sub-table under it //
// //////////////////////////////////////////////////////////////////////////
// if(!fieldName.contains("."))
// {
// fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue());
// }
// }
// }
// /////////////////////////////////////////////////////////////////////
// // loop over the length of the record, building associated records //
// /////////////////////////////////////////////////////////////////////
// QRecord associatedRecord = new QRecord();
// Set<String> processedFieldNames = new HashSet<>();
// boolean gotAnyValues = false;
// int subStartIndex = -1;
// for(int i = startIndex; i < endIndex; i++)
// {
// String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
// {
// ///////////////////////////////////////////////
// // ok - this is a value for this association //
// ///////////////////////////////////////////////
// if(subStartIndex == -1)
// {
// subStartIndex = i;
// }
// String fieldName = entry.getKey();
// if(processedFieldNames.contains(fieldName))
// {
// /////////////////////////////////////////////////
// // this means we're starting a new sub-record! //
// /////////////////////////////////////////////////
// if(gotAnyValues)
// {
// addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
// processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i);
// rs.add(associatedRecord);
// }
// associatedRecord = new QRecord();
// processedFieldNames = new HashSet<>();
// gotAnyValues = false;
// subStartIndex = i + 1;
// }
// processedFieldNames.add(fieldName);
// Serializable value = row.getValueElseNull(i);
// if(value != null && !"".equals(value))
// {
// gotAnyValues = true;
// }
// setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i);
// }
// }
// }
// ////////////////////////
// // handle final value //
// ////////////////////////
// if(gotAnyValues)
// {
// addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
// processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex);
// rs.add(associatedRecord);
// }
// return (rs);
// }
/***************************************************************************
**
***************************************************************************/
private void addDefaultValuesToAssociatedRecord(Set<String> processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain)
{
for(QFieldMetaData field : table.getFields().values())
{
if(!processedFieldNames.contains(field.getName()))
{
setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null);
}
}
}
}

78
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java
View File

@ -37,8 +37,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -48,7 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class WideRowsToRecord implements RowsToRecordInterface
public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
@ -77,7 +77,7 @@ public class WideRowsToRecord implements RowsToRecordInterface
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName()));
setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()));
}
processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size());
@ -85,7 +85,7 @@ public class WideRowsToRecord implements RowsToRecordInterface
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping);
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
@ -199,7 +199,7 @@ public class WideRowsToRecord implements RowsToRecordInterface
gotAnyValues = true;
}
setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i);
setValueOrDefault(associatedRecord, table.getField(fieldName), associationName, mapping, row, i);
}
}
}
@ -228,77 +228,11 @@ public class WideRowsToRecord implements RowsToRecordInterface
{
if(!processedFieldNames.contains(field.getName()))
{
setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null);
setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null);
}
}
}
/***************************************************************************
**
***************************************************************************/
// private List<QRecord> processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException
// {
// List<QRecord> rs = new ArrayList<>();
// String associationNameChainForRecursiveCalls = associationName;
// Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
// for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
// {
// if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + "."))
// {
// fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue());
// }
// }
// Map<String, List<Integer>> indexes = new HashMap<>();
// for(int i = 0; i < headerRow.size(); i++)
// {
// String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
// {
// indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i);
// }
// }
// }
// int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0);
// //////////////////////////////////////////////////////
// // figure out how many sub-rows we'll be processing //
// //////////////////////////////////////////////////////
// for(int i = 0; i < maxIndex; i++)
// {
// QRecord associatedRecord = new QRecord();
// boolean gotAnyValues = false;
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// String fieldName = entry.getKey();
// if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i)
// {
// Integer index = indexes.get(fieldName).get(i);
// Serializable value = row.getValueElseNull(index);
// if(value != null && !"".equals(value))
// {
// gotAnyValues = true;
// }
// setValueOrDefault(associatedRecord, fieldName, mapping, row, index);
// }
// }
// if(gotAnyValues)
// {
// processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size());
// rs.add(associatedRecord);
// }
// }
// return (rs);
// }
/***************************************************************************

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java
View File

@ -19,7 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable;

133
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java Normal file
View File

@ -0,0 +1,133 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable;
import java.util.ArrayList;
/***************************************************************************
* this is the model of a saved bulk load profile - which is what passes back
* and forth with the frontend.
****************************************************************************/
public class BulkLoadProfile implements Serializable
{
private ArrayList<BulkLoadProfileField> fieldList;
private Boolean hasHeaderRow;
private String layout;
/*******************************************************************************
** Getter for fieldList
*******************************************************************************/
public ArrayList<BulkLoadProfileField> getFieldList()
{
return (this.fieldList);
}
/*******************************************************************************
** Getter for hasHeaderRow
*******************************************************************************/
public Boolean getHasHeaderRow()
{
return (this.hasHeaderRow);
}
/*******************************************************************************
** Setter for hasHeaderRow
*******************************************************************************/
public void setHasHeaderRow(Boolean hasHeaderRow)
{
this.hasHeaderRow = hasHeaderRow;
}
/*******************************************************************************
** Fluent setter for hasHeaderRow
*******************************************************************************/
public BulkLoadProfile withHasHeaderRow(Boolean hasHeaderRow)
{
this.hasHeaderRow = hasHeaderRow;
return (this);
}
/*******************************************************************************
** Getter for layout
*******************************************************************************/
public String getLayout()
{
return (this.layout);
}
/*******************************************************************************
** Setter for layout
*******************************************************************************/
public void setLayout(String layout)
{
this.layout = layout;
}
/*******************************************************************************
** Fluent setter for layout
*******************************************************************************/
public BulkLoadProfile withLayout(String layout)
{
this.layout = layout;
return (this);
}
/*******************************************************************************
** Setter for fieldList
*******************************************************************************/
public void setFieldList(ArrayList<BulkLoadProfileField> fieldList)
{
this.fieldList = fieldList;
}
/*******************************************************************************
** Fluent setter for fieldList
*******************************************************************************/
public BulkLoadProfile withFieldList(ArrayList<BulkLoadProfileField> fieldList)
{
this.fieldList = fieldList;
return (this);
}
}

195
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java Normal file
View File

@ -0,0 +1,195 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable;
import java.util.Map;
/***************************************************************************
**
***************************************************************************/
public class BulkLoadProfileField
{
private String fieldName;
private Integer columnIndex;
private Serializable defaultValue;
private Boolean doValueMapping;
private Map<String, Serializable> valueMappings;
/*******************************************************************************
** Getter for fieldName
*******************************************************************************/
public String getFieldName()
{
return (this.fieldName);
}
/*******************************************************************************
** Setter for fieldName
*******************************************************************************/
public void setFieldName(String fieldName)
{
this.fieldName = fieldName;
}
/*******************************************************************************
** Fluent setter for fieldName
*******************************************************************************/
public BulkLoadProfileField withFieldName(String fieldName)
{
this.fieldName = fieldName;
return (this);
}
/*******************************************************************************
** Getter for columnIndex
*******************************************************************************/
public Integer getColumnIndex()
{
return (this.columnIndex);
}
/*******************************************************************************
** Setter for columnIndex
*******************************************************************************/
public void setColumnIndex(Integer columnIndex)
{
this.columnIndex = columnIndex;
}
/*******************************************************************************
** Fluent setter for columnIndex
*******************************************************************************/
public BulkLoadProfileField withColumnIndex(Integer columnIndex)
{
this.columnIndex = columnIndex;
return (this);
}
/*******************************************************************************
** Getter for defaultValue
*******************************************************************************/
public Serializable getDefaultValue()
{
return (this.defaultValue);
}
/*******************************************************************************
** Setter for defaultValue
*******************************************************************************/
public void setDefaultValue(Serializable defaultValue)
{
this.defaultValue = defaultValue;
}
/*******************************************************************************
** Fluent setter for defaultValue
*******************************************************************************/
public BulkLoadProfileField withDefaultValue(Serializable defaultValue)
{
this.defaultValue = defaultValue;
return (this);
}
/*******************************************************************************
** Getter for doValueMapping
*******************************************************************************/
public Boolean getDoValueMapping()
{
return (this.doValueMapping);
}
/*******************************************************************************
** Setter for doValueMapping
*******************************************************************************/
public void setDoValueMapping(Boolean doValueMapping)
{
this.doValueMapping = doValueMapping;
}
/*******************************************************************************
** Fluent setter for doValueMapping
*******************************************************************************/
public BulkLoadProfileField withDoValueMapping(Boolean doValueMapping)
{
this.doValueMapping = doValueMapping;
return (this);
}
/*******************************************************************************
** Getter for valueMappings
*******************************************************************************/
public Map<String, Serializable> getValueMappings()
{
return (this.valueMappings);
}
/*******************************************************************************
** Setter for valueMappings
*******************************************************************************/
public void setValueMappings(Map<String, Serializable> valueMappings)
{
this.valueMappings = valueMappings;
}
/*******************************************************************************
** Fluent setter for valueMappings
*******************************************************************************/
public BulkLoadProfileField withValueMappings(Map<String, Serializable> valueMappings)
{
this.valueMappings = valueMappings;
return (this);
}
}

275
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java Normal file
View File

@ -0,0 +1,275 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
/*******************************************************************************
**
*******************************************************************************/
public class BulkLoadTableStructure implements Serializable
{
private boolean isMain;
private boolean isMany;
private String tableName;
private String label;
private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild
private ArrayList<QFieldMetaData> fields; // mmm, not marked as serializable (at this time) - is okay?
private ArrayList<BulkLoadTableStructure> associations;
/*******************************************************************************
** Getter for isMain
*******************************************************************************/
public boolean getIsMain()
{
return (this.isMain);
}
/*******************************************************************************
** Setter for isMain
*******************************************************************************/
public void setIsMain(boolean isMain)
{
this.isMain = isMain;
}
/*******************************************************************************
** Fluent setter for isMain
*******************************************************************************/
public BulkLoadTableStructure withIsMain(boolean isMain)
{
this.isMain = isMain;
return (this);
}
/*******************************************************************************
** Getter for isMany
*******************************************************************************/
public boolean getIsMany()
{
return (this.isMany);
}
/*******************************************************************************
** Setter for isMany
*******************************************************************************/
public void setIsMany(boolean isMany)
{
this.isMany = isMany;
}
/*******************************************************************************
** Fluent setter for isMany
*******************************************************************************/
public BulkLoadTableStructure withIsMany(boolean isMany)
{
this.isMany = isMany;
return (this);
}
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getTableName()
{
return (this.tableName);
}
/*******************************************************************************
** Setter for tableName
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent setter for tableName
*******************************************************************************/
public BulkLoadTableStructure withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
/*******************************************************************************
** Getter for label
*******************************************************************************/
public String getLabel()
{
return (this.label);
}
/*******************************************************************************
** Setter for label
*******************************************************************************/
public void setLabel(String label)
{
this.label = label;
}
/*******************************************************************************
** Fluent setter for label
*******************************************************************************/
public BulkLoadTableStructure withLabel(String label)
{
this.label = label;
return (this);
}
/*******************************************************************************
** Getter for fields
*******************************************************************************/
public ArrayList<QFieldMetaData> getFields()
{
return (this.fields);
}
/*******************************************************************************
** Setter for fields
*******************************************************************************/
public void setFields(ArrayList<QFieldMetaData> fields)
{
this.fields = fields;
}
/*******************************************************************************
** Fluent setter for fields
*******************************************************************************/
public BulkLoadTableStructure withFields(ArrayList<QFieldMetaData> fields)
{
this.fields = fields;
return (this);
}
/*******************************************************************************
** Getter for associationPath
*******************************************************************************/
public String getAssociationPath()
{
return (this.associationPath);
}
/*******************************************************************************
** Setter for associationPath
*******************************************************************************/
public void setAssociationPath(String associationPath)
{
this.associationPath = associationPath;
}
/*******************************************************************************
** Fluent setter for associationPath
*******************************************************************************/
public BulkLoadTableStructure withAssociationPath(String associationPath)
{
this.associationPath = associationPath;
return (this);
}
/*******************************************************************************
** Getter for associations
*******************************************************************************/
public ArrayList<BulkLoadTableStructure> getAssociations()
{
return (this.associations);
}
/*******************************************************************************
** Setter for associations
*******************************************************************************/
public void setAssociations(ArrayList<BulkLoadTableStructure> associations)
{
this.associations = associations;
}
/*******************************************************************************
** Fluent setter for associations
*******************************************************************************/
public BulkLoadTableStructure withAssociations(ArrayList<BulkLoadTableStructure> associations)
{
this.associations = associations;
return (this);
}
/***************************************************************************
**
***************************************************************************/
public void addAssociation(BulkLoadTableStructure association)
{
if(this.associations == null)
{
this.associations = new ArrayList<>();
}
this.associations.add(association);
}
}