QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding unique key check to insert action; adding post-insert customizer

Browse Source
This commit is contained in:
darin.kelkhoff
2022-12-05 15:40:08 -06:00
parent c22fc89cbb
commit a769d8942c
19 changed files with 986 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryTableBackendDetails.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java
View File

@ -19,51 +19,47 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
package com.kingsrook.qqq.backend.core.actions.customizers;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
**
*******************************************************************************/
public class MemoryTableBackendDetails extends QTableBackendDetails
public abstract class AbstractPostInsertCustomizer
{
private boolean cloneUponStore = false;
protected InsertInput insertInput;
/*******************************************************************************
** Getter for cloneUponStore
**
*******************************************************************************/
public boolean getCloneUponStore()
public abstract List<QRecord> apply(List<QRecord> records);
/*******************************************************************************
** Getter for insertInput
**
*******************************************************************************/
public InsertInput getInsertInput()
{
return cloneUponStore;
return insertInput;
}
/*******************************************************************************
** Setter for cloneUponStore
** Setter for insertInput
**
*******************************************************************************/
public void setCloneUponStore(boolean cloneUponStore)
public void setInsertInput(InsertInput insertInput)
{
this.cloneUponStore = cloneUponStore;
this.insertInput = insertInput;
}
/*******************************************************************************
** Fluent setter for cloneUponStore
**
*******************************************************************************/
public MemoryTableBackendDetails withCloneUponStore(boolean cloneUponStore)
{
this.cloneUponStore = cloneUponStore;
return (this);
}
}

160
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java Normal file
View File

@ -0,0 +1,160 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.customizers;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Standard/re-usable post-insert customizer, for the use case where, when we
** do an insert into table "parent", we want a record automatically inserted into
** table "child", and there's a foreign key in "parent", pointed at "child"
** e.g., named: "parent.childId".
**
** A similar use-case would have the foreign key in the child table - in which case,
** we could add a "Type" enum, plus abstract method to get our "Type", then logic
** to switch behavior based on type. See existing type enum, but w/ only 1 case :)
*******************************************************************************/
public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer
{
public enum RelationshipType
{
PARENT_POINTS_AT_CHILD
}
/*******************************************************************************
**
*******************************************************************************/
public abstract QRecord buildChildForRecord(QRecord parentRecord) throws QException;
/*******************************************************************************
**
*******************************************************************************/
public abstract String getChildTableName();
/*******************************************************************************
**
*******************************************************************************/
public abstract String getForeignKeyFieldName();
/*******************************************************************************
**
*******************************************************************************/
public abstract RelationshipType getRelationshipType();
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> apply(List<QRecord> records)
{
try
{
List<QRecord> rs = new ArrayList<>();
List<QRecord> childrenToInsert = new ArrayList<>();
QTableMetaData table = getInsertInput().getTable();
QTableMetaData childTable = getInsertInput().getInstance().getTable(getChildTableName());
////////////////////////////////////////////////////////////////////////////////
// iterate over the inserted records, building a list child records to insert //
// for ones missing a value in the foreign key field. //
////////////////////////////////////////////////////////////////////////////////
for(QRecord record : records)
{
if(record.getValue(getForeignKeyFieldName()) == null)
{
childrenToInsert.add(buildChildForRecord(record));
}
}
///////////////////////////////////////////////////////////////////////////////////
// if there are no children to insert, then just return the original record list //
///////////////////////////////////////////////////////////////////////////////////
if(childrenToInsert.isEmpty())
{
return (records);
}
/////////////////////////
// insert the children //
/////////////////////////
InsertInput insertInput = new InsertInput(getInsertInput().getInstance());
insertInput.setSession(getInsertInput().getSession());
insertInput.setTableName(getChildTableName());
insertInput.setRecords(childrenToInsert);
InsertOutput insertOutput = new InsertAction().execute(insertInput);
Iterator<QRecord> insertedRecordIterator = insertOutput.getRecords().iterator();
//////////////////////////////////////////////////////////////////////////////////////////////////////
// iterate over the original list of records again - for any that need a child (e.g., are missing //
// foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. //
//////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsToUpdate = new ArrayList<>();
for(QRecord record : records)
{
Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
if(record.getValue(getForeignKeyFieldName()) == null)
{
Serializable foreignKey = insertedRecordIterator.next().getValue(childTable.getPrimaryKeyField());
recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
record.setValue(getForeignKeyFieldName(), foreignKey);
rs.add(record);
}
else
{
rs.add(record);
}
}
////////////////////////////////////////////////////////////////////////////
// update the originally inserted records to reference their new children //
////////////////////////////////////////////////////////////////////////////
UpdateInput updateInput = new UpdateInput(insertInput.getInstance());
updateInput.setSession(getInsertInput().getSession());
updateInput.setTableName(getInsertInput().getTableName());
updateInput.setRecords(recordsToUpdate);
new UpdateAction().execute(updateInput);
return (rs);
}
catch(Exception e)
{
throw new RuntimeException("Error inserting new child records for new parent records", e);
}
}
}

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java
View File

@ -61,6 +61,21 @@ public class QCodeLoader
/*******************************************************************************
**
*******************************************************************************/
public static <T> Optional<T> getTableCustomizer(Class<T> expectedClass, QTableMetaData table, String customizerName)
{
Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
if(codeReference.isPresent())
{
return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get())));
}
return (Optional.empty());
}
/*******************************************************************************
**
*******************************************************************************/

12
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java
View File

@ -49,6 +49,18 @@ public class TableCustomizer
/*******************************************************************************
**
*******************************************************************************/
public TableCustomizer(String role, Class<?> expectedType)
{
this.role = role;
this.expectedType = expectedType;
this.validationFunction = null;
}
/*******************************************************************************
** Getter for role
**

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java
View File

@ -47,8 +47,10 @@ public enum TableCustomizers
{
@SuppressWarnings("unchecked")
Function<QRecord, QRecord> function = (Function<QRecord, QRecord>) x;
QRecord output = function.apply(new QRecord());
})));
QRecord output = function.apply(new QRecord());
}))),
POST_INSERT_RECORD(new TableCustomizer("postInsertRecord", AbstractPostInsertCustomizer.class));
private final TableCustomizer tableCustomizer;

93
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -22,17 +22,32 @@
package com.kingsrook.qqq.backend.core.actions.tables;
import java.io.Serializable;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -54,20 +69,96 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
public InsertOutput execute(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);
QTableMetaData table = insertInput.getTable();
Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
setAutomationStatusField(insertInput);
ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), insertInput.getTable(), insertInput.getRecords());
ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
// todo - need to handle records with errors coming out of here...
QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
// todo pre-customization - just get to modify the request?
setErrorsIfUniqueKeyErrors(insertInput, table);
InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput);
// todo post-customization - can do whatever w/ the result if you want
if(postInsertCustomizer.isPresent())
{
postInsertCustomizer.get().setInsertInput(insertInput);
insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
}
return insertOutput;
}
/*******************************************************************************
**
*******************************************************************************/
private void setErrorsIfUniqueKeyErrors(InsertInput insertInput, QTableMetaData table) throws QException
{
if(CollectionUtils.nullSafeHasContents(table.getUniqueKeys()))
{
Map<UniqueKey, Set<List<Serializable>>> keysInThisList = new HashMap<>();
if(insertInput.getSkipUniqueKeyCheck())
{
LOG.debug("Skipping unique key check in " + insertInput.getTableName() + " insert.");
return;
}
////////////////////////////////////////////
// check for any pre-existing unique keys //
////////////////////////////////////////////
Map<UniqueKey, Set<List<Serializable>>> existingKeys = new HashMap<>();
List<UniqueKey> uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys());
for(UniqueKey uniqueKey : uniqueKeys)
{
existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(insertInput, insertInput.getTransaction(), table, insertInput.getRecords(), uniqueKey));
}
/////////////////////////////////////
// make sure this map is populated //
/////////////////////////////////////
uniqueKeys.forEach(uk -> keysInThisList.computeIfAbsent(uk, x -> new HashSet<>()));
for(QRecord record : insertInput.getRecords())
{
//////////////////////////////////////////////////////////
// check if this record violates any of the unique keys //
//////////////////////////////////////////////////////////
boolean foundDupe = false;
for(UniqueKey uniqueKey : uniqueKeys)
{
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get())))
{
record.addError("Another record already exists with this " + uniqueKey.getDescription(table));
foundDupe = true;
break;
}
}
///////////////////////////////////////////////////////////////////////////////
// if this record doesn't violate any uk's, then we can add it to the output //
///////////////////////////////////////////////////////////////////////////////
if(!foundDupe)
{
for(UniqueKey uniqueKey : uniqueKeys)
{
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
keyValues.ifPresent(kv -> keysInThisList.get(uniqueKey).add(kv));
}
}
}
}
}
/*******************************************************************************
** If the table being inserted into uses an automation-status field, populate it now.
*******************************************************************************/

171
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java Normal file
View File

@ -0,0 +1,171 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** Methods to help with unique key checks.
*******************************************************************************/
public class UniqueKeyHelper
{
/*******************************************************************************
**
*******************************************************************************/
public static Set<List<Serializable>> getExistingKeys(AbstractActionInput actionInput, QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
{
List<String> ukFieldNames = uniqueKey.getFieldNames();
Set<List<Serializable>> existingRecords = new HashSet<>();
if(ukFieldNames != null)
{
QueryInput queryInput = new QueryInput(actionInput.getInstance());
queryInput.setSession(actionInput.getSession());
queryInput.setTableName(table.getName());
queryInput.setTransaction(transaction);
QQueryFilter filter = new QQueryFilter();
if(ukFieldNames.size() == 1)
{
List<Serializable> values = recordList.stream()
.filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
.map(r -> r.getValue(ukFieldNames.get(0)))
.collect(Collectors.toList());
filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
}
else
{
filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
for(QRecord record : recordList)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
continue;
}
QQueryFilter subFilter = new QQueryFilter();
filter.addSubFilter(subFilter);
for(String fieldName : ukFieldNames)
{
Serializable value = record.getValue(fieldName);
if(value == null)
{
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
}
else
{
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
}
}
}
if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - rather - return early. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
return (existingRecords);
}
}
queryInput.setFilter(filter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
for(QRecord record : queryOutput.getRecords())
{
Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record);
keyValues.ifPresent(existingRecords::add);
}
}
return (existingRecords);
}
/*******************************************************************************
**
*******************************************************************************/
public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
{
try
{
List<Serializable> keyValues = new ArrayList<>();
for(String fieldName : uniqueKey.getFieldNames())
{
QFieldMetaData field = table.getField(fieldName);
Serializable value = record.getValue(fieldName);
Serializable typedValue = ValueUtils.getValueAsFieldType(field.getType(), value);
keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue);
}
return (Optional.of(keyValues));
}
catch(Exception e)
{
return (Optional.empty());
}
}
/*******************************************************************************
** To make a list of unique key values here behave like they do in an RDBMS
** (which is what we're trying to mimic - which is - 2 null values in a field
** aren't considered the same, so they don't violate a unique key) (at least, that's
** how some RDBMS's work, right??) - use this value instead of nulls in the
** output of getKeyValues - where interestingly, this class always returns
** false in it equals method... Unclear how bad this is, e.g., if it's violating
** the contract for equals and hashCode...
*******************************************************************************/
public static class NullUniqueKeyValue implements Serializable
{
@Override
public boolean equals(Object obj)
{
return (false);
}
}
}

40
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java
View File

@ -36,7 +36,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
public class InsertInput extends AbstractTableActionInput
{
private QBackendTransaction transaction;
private List<QRecord> records;
private List<QRecord> records;
private boolean skipUniqueKeyCheck = false;
@ -80,6 +82,7 @@ public class InsertInput extends AbstractTableActionInput
}
/*******************************************************************************
** Fluent setter for transaction
**
@ -111,4 +114,39 @@ public class InsertInput extends AbstractTableActionInput
{
this.records = records;
}
/*******************************************************************************
** Getter for skipUniqueKeyCheck
**
*******************************************************************************/
public boolean getSkipUniqueKeyCheck()
{
return skipUniqueKeyCheck;
}
/*******************************************************************************
** Setter for skipUniqueKeyCheck
**
*******************************************************************************/
public void setSkipUniqueKeyCheck(boolean skipUniqueKeyCheck)
{
this.skipUniqueKeyCheck = skipUniqueKeyCheck;
}
/*******************************************************************************
** Fluent setter for skipUniqueKeyCheck
**
*******************************************************************************/
public InsertInput withSkipUniqueKeyCheck(boolean skipUniqueKeyCheck)
{
this.skipUniqueKeyCheck = skipUniqueKeyCheck;
return (this);
}
}

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java
View File

@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
@ -182,6 +183,12 @@ public class MemoryRecordStore
QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
for(QRecord record : input.getRecords())
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
outputRecords.add(record);
continue;
}
/////////////////////////////////////////////////
// set the next serial in the record if needed //
/////////////////////////////////////////////////

100
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -28,24 +28,20 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -71,6 +67,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
this.table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue(LoadViaInsertStep.FIELD_SKIP_UNIQUE_KEY_CHECK, true);
}
@ -87,7 +88,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
List<UniqueKey> uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys());
for(UniqueKey uniqueKey : uniqueKeys)
{
existingKeys.put(uniqueKey, getExistingKeys(runBackendStepInput, uniqueKey));
existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(runBackendStepInput, null, table, runBackendStepInput.getRecords(), uniqueKey));
ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLine(Status.ERROR));
}
@ -142,8 +143,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
boolean foundDupe = false;
for(UniqueKey uniqueKey : uniqueKeys)
{
List<Serializable> keyValues = getKeyValues(uniqueKey, record);
if(existingKeys.get(uniqueKey).contains(keyValues) || keysInThisFile.get(uniqueKey).contains(keyValues))
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get())))
{
ukErrorSummaries.get(uniqueKey).incrementCount();
foundDupe = true;
@ -158,8 +159,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
{
for(UniqueKey uniqueKey : uniqueKeys)
{
List<Serializable> keyValues = getKeyValues(uniqueKey, record);
keysInThisFile.get(uniqueKey).add(keyValues);
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv));
}
okSummary.incrementCount();
runBackendStepOutput.addRecord(record);
@ -170,81 +171,6 @@ public class BulkInsertTransformStep extends AbstractTransformStep
/*******************************************************************************
**
*******************************************************************************/
private Set<List<Serializable>> getExistingKeys(RunBackendStepInput runBackendStepInput, UniqueKey uniqueKey) throws QException
{
List<String> ukFieldNames = uniqueKey.getFieldNames();
Set<List<Serializable>> existingRecords = new HashSet<>();
if(ukFieldNames != null)
{
QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance());
queryInput.setSession(runBackendStepInput.getSession());
queryInput.setTableName(runBackendStepInput.getTableName());
getTransaction().ifPresent(queryInput::setTransaction);
QQueryFilter filter = new QQueryFilter();
if(ukFieldNames.size() == 1)
{
List<Serializable> values = runBackendStepInput.getRecords().stream()
.map(r -> r.getValue(ukFieldNames.get(0)))
.collect(Collectors.toList());
filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
}
else
{
filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
for(QRecord record : runBackendStepInput.getRecords())
{
QQueryFilter subFilter = new QQueryFilter();
filter.addSubFilter(subFilter);
for(String fieldName : ukFieldNames)
{
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, List.of(record.getValue(fieldName))));
}
}
}
queryInput.setFilter(filter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
for(QRecord record : queryOutput.getRecords())
{
List<Serializable> keyValues = getKeyValues(ukFieldNames, record);
existingRecords.add(keyValues);
}
}
return (existingRecords);
}
/*******************************************************************************
**
*******************************************************************************/
private List<Serializable> getKeyValues(UniqueKey uniqueKey, QRecord record)
{
return (getKeyValues(uniqueKey.getFieldNames(), record));
}
/*******************************************************************************
**
*******************************************************************************/
private List<Serializable> getKeyValues(List<String> fieldNames, QRecord record)
{
List<Serializable> keyValues = new ArrayList<>();
for(String fieldName : fieldNames)
{
keyValues.add(record.getValue(fieldName));
}
return keyValues;
}
/*******************************************************************************
**
*******************************************************************************/

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java
View File

@ -48,7 +48,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
*******************************************************************************/
public class LoadViaInsertOrUpdateStep extends AbstractLoadStep
{
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
public static final String FIELD_SKIP_UNIQUE_KEY_CHECK = "skipUniqueKeyCheck";
protected List<QRecord> recordsToInsert = null;
protected List<QRecord> recordsToUpdate = null;
@ -83,6 +84,12 @@ public class LoadViaInsertOrUpdateStep extends AbstractLoadStep
insertInput.setRecords(recordsToInsert);
getTransaction().ifPresent(insertInput::setTransaction);
insertInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
if(runBackendStepInput.getValuePrimitiveBoolean(FIELD_SKIP_UNIQUE_KEY_CHECK))
{
insertInput.setSkipUniqueKeyCheck(true);
}
InsertOutput insertOutput = new InsertAction().execute(insertInput);
runBackendStepOutput.getRecords().addAll(insertOutput.getRecords());
}

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java
View File

@ -38,7 +38,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
*******************************************************************************/
public class LoadViaInsertStep extends AbstractLoadStep
{
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
public static final String FIELD_SKIP_UNIQUE_KEY_CHECK = "skipUniqueKeyCheck";
@ -55,6 +56,12 @@ public class LoadViaInsertStep extends AbstractLoadStep
insertInput.setRecords(runBackendStepInput.getRecords());
getTransaction().ifPresent(insertInput::setTransaction);
insertInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
if(runBackendStepInput.getValuePrimitiveBoolean(FIELD_SKIP_UNIQUE_KEY_CHECK))
{
insertInput.setSkipUniqueKeyCheck(true);
}
InsertOutput insertOutput = new InsertAction().execute(insertInput);
runBackendStepOutput.getRecords().addAll(insertOutput.getRecords());
}

21
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java
View File

@ -38,6 +38,7 @@ import java.util.Calendar;
import java.util.List;
import java.util.TimeZone;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
/*******************************************************************************
@ -632,4 +633,24 @@ public class ValueUtils
}
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("checkstyle:indentation")
public static Serializable getValueAsFieldType(QFieldType type, Serializable value)
{
return switch(type)
{
case STRING, TEXT, HTML, PASSWORD -> getValueAsString(value);
case INTEGER -> getValueAsInteger(value);
case DECIMAL -> getValueAsBigDecimal(value);
case BOOLEAN -> getValueAsBoolean(value);
case DATE -> getValueAsLocalDate(value);
case TIME -> getValueAsLocalTime(value);
case DATE_TIME -> getValueAsInstant(value);
case BLOB -> getValueAsByteArray(value);
};
}
}

221
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java Normal file
View File

@ -0,0 +1,221 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.customizers;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for ChildInserterPostInsertCustomizer
**
** We'll use person & shape tables here, w/ the favoriteShapeId foreign key,
** so a rule of "every time we insert a person, if they aren't already pointed at
** a favoriteShape, insert a new shape for them".
*******************************************************************************/
class ChildInserterPostInsertCustomizerTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testEmptyCases() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
addPostInsertActionToTable(qInstance);
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of());
////////////////////////////////////////
// just looking for no exception here //
////////////////////////////////////////
new InsertAction().execute(insertInput);
assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// now insert one person, but they shouldn't need a favoriteShape to be inserted - again, make sure we don't blow up and that no shapes get inserted. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
insertInput.setRecords(List.of(new QRecord().withValue("firstName", "James").withValue("favoriteShapeId", -1)));
new InsertAction().execute(insertInput);
assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
}
/*******************************************************************************
**
*******************************************************************************/
private static void addPostInsertActionToTable(QInstance qInstance)
{
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withCustomizer(TableCustomizers.POST_INSERT_RECORD.getTableCustomizer(), new QCodeReference(PersonPostInsertAddFavoriteShapeCustomizer.class, QCodeUsage.CUSTOMIZER));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSimpleCase() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
addPostInsertActionToTable(qInstance);
assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("firstName", "Darin")
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
Serializable favoriteShapeId = insertOutput.getRecords().get(0).getValue("favoriteShapeId");
assertNotNull(favoriteShapeId);
List<QRecord> shapeRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE);
assertEquals(1, shapeRecords.size());
assertEquals(favoriteShapeId, shapeRecords.get(0).getValue("id"));
assertEquals("Darin's favorite shape!", shapeRecords.get(0).getValue("name"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testComplexCase() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
addPostInsertActionToTable(qInstance);
assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("firstName", "Darin"),
new QRecord().withValue("firstName", "James").withValue("favoriteShapeId", -1),
new QRecord().withValue("firstName", "Tim"),
new QRecord().withValue("firstName", "Garret").withValue("favoriteShapeId", -2)
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
assertEquals(1, insertOutput.getRecords().get(0).getValue("favoriteShapeId"));
assertEquals(-1, insertOutput.getRecords().get(1).getValue("favoriteShapeId"));
assertEquals(2, insertOutput.getRecords().get(2).getValue("favoriteShapeId"));
assertEquals(-2, insertOutput.getRecords().get(3).getValue("favoriteShapeId"));
List<QRecord> shapeRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE);
assertEquals(2, shapeRecords.size());
assertEquals(1, shapeRecords.get(0).getValue("id"));
assertEquals(2, shapeRecords.get(1).getValue("id"));
assertEquals("Darin's favorite shape!", shapeRecords.get(0).getValue("name"));
assertEquals("Tim's favorite shape!", shapeRecords.get(1).getValue("name"));
}
/*******************************************************************************
**
*******************************************************************************/
public static class PersonPostInsertAddFavoriteShapeCustomizer extends ChildInserterPostInsertCustomizer
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public QRecord buildChildForRecord(QRecord parentRecord) throws QException
{
return (new QRecord().withValue("name", parentRecord.getValue("firstName") + "'s favorite shape!"));
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getChildTableName()
{
return (TestUtils.TABLE_NAME_SHAPE);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getForeignKeyFieldName()
{
return ("favoriteShapeId");
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public RelationshipType getRelationshipType()
{
return (RelationshipType.PARENT_POINTS_AT_CHILD);
}
}
}

157
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java
View File

@ -28,9 +28,20 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
@ -40,6 +51,18 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
class InsertActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
** At the core level, there isn't much that can be asserted, as it uses the
** mock implementation - just confirming that all of the "wiring" works.
@ -51,8 +74,8 @@ class InsertActionTest
InsertInput request = new InsertInput(TestUtils.defineInstance());
request.setSession(TestUtils.getMockSession());
request.setTableName("person");
List<QRecord> records =new ArrayList<>();
QRecord record = new QRecord();
List<QRecord> records = new ArrayList<>();
QRecord record = new QRecord();
record.setValue("firstName", "James");
records.add(record);
request.setRecords(records);
@ -60,4 +83,134 @@ class InsertActionTest
assertNotNull(result);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKeysPreExisting() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff")
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
///////////////////////////////////////////////////////
// try to insert that person again - shouldn't work. //
///////////////////////////////////////////////////////
insertInput.setRecords(List.of(
new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff")
));
insertOutput = new InsertAction().execute(insertInput);
assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
assertNull(insertOutput.getRecords().get(0).getValueInteger("id"));
assertEquals(1, insertOutput.getRecords().get(0).getErrors().size());
assertThat(insertOutput.getRecords().get(0).getErrors().get(0)).contains("Another record already exists with this First Name and Last Name");
//////////////////////////////////////////////////////////////////////////////////////////
// try to insert that person again, with 2 others - the 2 should work, but the one fail //
//////////////////////////////////////////////////////////////////////////////////////////
insertInput.setRecords(List.of(
new QRecord().withValue("firstName", "Darin").withValue("lastName", "Smith"),
new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"),
new QRecord().withValue("firstName", "Trevor").withValue("lastName", "Kelkhoff")
));
insertOutput = new InsertAction().execute(insertInput);
assertEquals(3, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
assertNotNull(insertOutput.getRecords().get(0).getValueInteger("id"));
assertNull(insertOutput.getRecords().get(1).getValueInteger("id"));
assertNotNull(insertOutput.getRecords().get(2).getValueInteger("id"));
assertEquals(0, insertOutput.getRecords().get(0).getErrors().size());
assertEquals(1, insertOutput.getRecords().get(1).getErrors().size());
assertEquals(0, insertOutput.getRecords().get(2).getErrors().size());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKeysWithinBatch() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"),
new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff")
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
assertEquals(1, insertOutput.getRecords().get(0).getValueInteger("id"));
assertNull(insertOutput.getRecords().get(1).getValueInteger("id"));
assertEquals(1, insertOutput.getRecords().get(1).getErrors().size());
assertThat(insertOutput.getRecords().get(1).getErrors().get(0)).contains("Another record already exists with this First Name and Last Name");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSingleColumnUniqueKey() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.getTable(TestUtils.TABLE_NAME_SHAPE)
.withUniqueKey(new UniqueKey("name"));
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE);
insertInput.setRecords(List.of(
new QRecord().withValue("name", "Circle"),
new QRecord().withValue("name", "Circle")
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
assertEquals(1, insertOutput.getRecords().get(0).getValueInteger("id"));
assertNull(insertOutput.getRecords().get(1).getValueInteger("id"));
assertEquals(1, insertOutput.getRecords().get(1).getErrors().size());
assertThat(insertOutput.getRecords().get(1).getErrors().get(0)).contains("Another record already exists with this Name");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSkippingUniqueKeys() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setSkipUniqueKeyCheck(true);
insertInput.setRecords(List.of(
new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"),
new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff")
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
assertEquals(2, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
assertEquals(1, insertOutput.getRecords().get(0).getValueInteger("id"));
assertEquals(2, insertOutput.getRecords().get(1).getValueInteger("id"));
assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(1).getErrors()));
}
}

3
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -98,7 +98,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
@ -652,8 +651,6 @@ public class TestUtils
return (new QTableMetaData()
.withName(TABLE_NAME_PERSON_MEMORY_CACHE)
.withBackendName(MEMORY_BACKEND_NAME)
.withBackendDetails(new MemoryTableBackendDetails()
.withCloneUponStore(true))
.withPrimaryKeyField("id")
.withUniqueKey(uniqueKey)
.withFields(TestUtils.defineTablePerson().getFields()))

37
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
View File

@ -114,13 +114,25 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
List<Object> params = new ArrayList<>();
int recordIndex = 0;
//////////////////////////////////////////////////////
// for each record in the page: //
// - if it has errors, skip it //
// - else add a "(?,?,...,?)," clause to the INSERT //
// - then add all fields into the params list //
//////////////////////////////////////////////////////
for(QRecord record : page)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
continue;
}
if(recordIndex++ > 0)
{
sql.append(",");
}
sql.append("(").append(questionMarks).append(")");
for(QFieldMetaData field : insertableFields)
{
Serializable value = record.getValue(field.getName());
@ -129,6 +141,23 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
}
}
////////////////////////////////////////////////////////////////////////////////////////
// if all records had errors, copy them to the output, and continue w/o running query //
////////////////////////////////////////////////////////////////////////////////////////
if(recordIndex == 0)
{
for(QRecord record : page)
{
QRecord outputRecord = new QRecord(record);
outputRecords.add(outputRecord);
}
continue;
}
///////////////////////////////////////////////////////////
// execute the insert, then foreach record in the input, //
// add it to the output, and set its generated id too. //
///////////////////////////////////////////////////////////
// todo sql customization - can edit sql and/or param list
// todo - non-serial-id style tables
// todo - other generated values, e.g., createDate... maybe need to re-select?
@ -136,10 +165,14 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
int index = 0;
for(QRecord record : page)
{
Integer id = idList.get(index++);
QRecord outputRecord = new QRecord(record);
outputRecord.setValue(table.getPrimaryKeyField(), id);
outputRecords.add(outputRecord);
if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
{
Integer id = idList.get(index++);
outputRecord.setValue(table.getPrimaryKeyField(), id);
}
}
}
}

2
qqq-dev-tools/CURRENT-SNAPSHOT-VERSION
View File

@ -1 +1 @@
0.8.0
0.9.0

5
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -522,6 +522,11 @@ public class QJavalinImplementation
InsertAction insertAction = new InsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors()))
{
throw (new QUserFacingException("Error inserting " + qInstance.getTable(table).getLabel() + ": " + insertOutput.getRecords().get(0).getErrors().get(0)));
}
context.result(JsonUtils.toJson(insertOutput));
}
catch(Exception e)