Adding unique key check to insert action; adding post-insert customizer

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java

@@ -19,51 +19,47 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
+package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class MemoryTableBackendDetails extends QTableBackendDetails
+public abstract class AbstractPostInsertCustomizer
 {
-   private boolean cloneUponStore = false;
+   protected InsertInput insertInput;
 
 
 
    /*******************************************************************************
-    ** Getter for cloneUponStore
     **
     *******************************************************************************/
-   public boolean getCloneUponStore()
+   public abstract List<QRecord> apply(List<QRecord> records);
+
+
+
+   /*******************************************************************************
+    ** Getter for insertInput
+    **
+    *******************************************************************************/
+   public InsertInput getInsertInput()
    {
-      return cloneUponStore;
+      return insertInput;
    }
 
 
 
    /*******************************************************************************
-    ** Setter for cloneUponStore
+    ** Setter for insertInput
     **
     *******************************************************************************/
-   public void setCloneUponStore(boolean cloneUponStore)
+   public void setInsertInput(InsertInput insertInput)
    {
-      this.cloneUponStore = cloneUponStore;
+      this.insertInput = insertInput;
    }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for cloneUponStore
-    **
-    *******************************************************************************/
-   public MemoryTableBackendDetails withCloneUponStore(boolean cloneUponStore)
-   {
-      this.cloneUponStore = cloneUponStore;
-      return (this);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java

@@ -0,0 +1,160 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Standard/re-usable post-insert customizer, for the use case where, when we
+ ** do an insert into table "parent", we want a record automatically inserted into
+ ** table "child", and there's a foreign key in "parent", pointed at "child"
+ ** e.g., named: "parent.childId".
+ **
+ ** A similar use-case would have the foreign key in the child table - in which case,
+ ** we could add a "Type" enum, plus abstract method to get our "Type", then logic
+ ** to switch behavior based on type.  See existing type enum, but w/ only 1 case :)
+ *******************************************************************************/
+public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer
+{
+   public enum RelationshipType
+   {
+      PARENT_POINTS_AT_CHILD
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract QRecord buildChildForRecord(QRecord parentRecord) throws QException;
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract String getChildTableName();
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract String getForeignKeyFieldName();
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract RelationshipType getRelationshipType();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> apply(List<QRecord> records)
+   {
+      try
+      {
+         List<QRecord>  rs               = new ArrayList<>();
+         List<QRecord>  childrenToInsert = new ArrayList<>();
+         QTableMetaData table            = getInsertInput().getTable();
+         QTableMetaData childTable       = getInsertInput().getInstance().getTable(getChildTableName());
+
+         ////////////////////////////////////////////////////////////////////////////////
+         // iterate over the inserted records, building a list child records to insert //
+         // for ones missing a value in the foreign key field.                         //
+         ////////////////////////////////////////////////////////////////////////////////
+         for(QRecord record : records)
+         {
+            if(record.getValue(getForeignKeyFieldName()) == null)
+            {
+               childrenToInsert.add(buildChildForRecord(record));
+            }
+         }
+
+         ///////////////////////////////////////////////////////////////////////////////////
+         // if there are no children to insert, then just return the original record list //
+         ///////////////////////////////////////////////////////////////////////////////////
+         if(childrenToInsert.isEmpty())
+         {
+            return (records);
+         }
+
+         /////////////////////////
+         // insert the children //
+         /////////////////////////
+         InsertInput insertInput = new InsertInput(getInsertInput().getInstance());
+         insertInput.setSession(getInsertInput().getSession());
+         insertInput.setTableName(getChildTableName());
+         insertInput.setRecords(childrenToInsert);
+         InsertOutput      insertOutput           = new InsertAction().execute(insertInput);
+         Iterator<QRecord> insertedRecordIterator = insertOutput.getRecords().iterator();
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////
+         // iterate over the original list of records again - for any that need a child (e.g., are missing   //
+         // foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QRecord> recordsToUpdate = new ArrayList<>();
+         for(QRecord record : records)
+         {
+            Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
+            if(record.getValue(getForeignKeyFieldName()) == null)
+            {
+               Serializable foreignKey = insertedRecordIterator.next().getValue(childTable.getPrimaryKeyField());
+               recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
+               record.setValue(getForeignKeyFieldName(), foreignKey);
+               rs.add(record);
+            }
+            else
+            {
+               rs.add(record);
+            }
+         }
+
+         ////////////////////////////////////////////////////////////////////////////
+         // update the originally inserted records to reference their new children //
+         ////////////////////////////////////////////////////////////////////////////
+         UpdateInput updateInput = new UpdateInput(insertInput.getInstance());
+         updateInput.setSession(getInsertInput().getSession());
+         updateInput.setTableName(getInsertInput().getTableName());
+         updateInput.setRecords(recordsToUpdate);
+         new UpdateAction().execute(updateInput);
+
+         return (rs);
+      }
+      catch(Exception e)
+      {
+         throw new RuntimeException("Error inserting new child records for new parent records", e);
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java

@@ -61,6 +61,21 @@ public class QCodeLoader
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static <T> Optional<T> getTableCustomizer(Class<T> expectedClass, QTableMetaData table, String customizerName)
+   {
+      Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
+      if(codeReference.isPresent())
+      {
+         return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get())));
+      }
+      return (Optional.empty());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java

@@ -49,6 +49,18 @@ public class TableCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public TableCustomizer(String role, Class<?> expectedType)
+   {
+      this.role = role;
+      this.expectedType = expectedType;
+      this.validationFunction = null;
+   }
+
+
+
    /*******************************************************************************
     ** Getter for role
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java

@@ -47,8 +47,10 @@ public enum TableCustomizers
    {
       @SuppressWarnings("unchecked")
       Function<QRecord, QRecord> function = (Function<QRecord, QRecord>) x;
-      QRecord                    output   = function.apply(new QRecord());
+      QRecord output = function.apply(new QRecord());
-   })));
+   }))),
+
+   POST_INSERT_RECORD(new TableCustomizer("postInsertRecord", AbstractPostInsertCustomizer.class));
 
 
    private final TableCustomizer tableCustomizer;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -22,17 +22,32 @@
 package com.kingsrook.qqq.backend.core.actions.tables;
 
 
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -54,20 +69,96 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    public InsertOutput execute(InsertInput insertInput) throws QException
    {
       ActionHelper.validateSession(insertInput);
+      QTableMetaData table = insertInput.getTable();
+
+      Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
       setAutomationStatusField(insertInput);
 
-      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), insertInput.getTable(), insertInput.getRecords());
+      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
       // todo - need to handle records with errors coming out of here...
 
       QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
       // todo pre-customization - just get to modify the request?
+
+      setErrorsIfUniqueKeyErrors(insertInput, table);
+
       InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput);
       // todo post-customization - can do whatever w/ the result if you want
+
+      if(postInsertCustomizer.isPresent())
+      {
+         postInsertCustomizer.get().setInsertInput(insertInput);
+         insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
+      }
+
       return insertOutput;
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setErrorsIfUniqueKeyErrors(InsertInput insertInput, QTableMetaData table) throws QException
+   {
+      if(CollectionUtils.nullSafeHasContents(table.getUniqueKeys()))
+      {
+         Map<UniqueKey, Set<List<Serializable>>> keysInThisList = new HashMap<>();
+         if(insertInput.getSkipUniqueKeyCheck())
+         {
+            LOG.debug("Skipping unique key check in " + insertInput.getTableName() + " insert.");
+            return;
+         }
+
+         ////////////////////////////////////////////
+         // check for any pre-existing unique keys //
+         ////////////////////////////////////////////
+         Map<UniqueKey, Set<List<Serializable>>> existingKeys = new HashMap<>();
+         List<UniqueKey>                         uniqueKeys   = CollectionUtils.nonNullList(table.getUniqueKeys());
+         for(UniqueKey uniqueKey : uniqueKeys)
+         {
+            existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(insertInput, insertInput.getTransaction(), table, insertInput.getRecords(), uniqueKey));
+         }
+
+         /////////////////////////////////////
+         // make sure this map is populated //
+         /////////////////////////////////////
+         uniqueKeys.forEach(uk -> keysInThisList.computeIfAbsent(uk, x -> new HashSet<>()));
+
+         for(QRecord record : insertInput.getRecords())
+         {
+            //////////////////////////////////////////////////////////
+            // check if this record violates any of the unique keys //
+            //////////////////////////////////////////////////////////
+            boolean foundDupe = false;
+            for(UniqueKey uniqueKey : uniqueKeys)
+            {
+               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
+               if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get())))
+               {
+                  record.addError("Another record already exists with this " + uniqueKey.getDescription(table));
+                  foundDupe = true;
+                  break;
+               }
+            }
+
+            ///////////////////////////////////////////////////////////////////////////////
+            // if this record doesn't violate any uk's, then we can add it to the output //
+            ///////////////////////////////////////////////////////////////////////////////
+            if(!foundDupe)
+            {
+               for(UniqueKey uniqueKey : uniqueKeys)
+               {
+                  Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
+                  keyValues.ifPresent(kv -> keysInThisList.get(uniqueKey).add(kv));
+               }
+            }
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     ** If the table being inserted into uses an automation-status field, populate it now.
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java

@@ -0,0 +1,171 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Methods to help with unique key checks.
+ *******************************************************************************/
+public class UniqueKeyHelper
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Set<List<Serializable>> getExistingKeys(AbstractActionInput actionInput, QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
+   {
+      List<String>            ukFieldNames    = uniqueKey.getFieldNames();
+      Set<List<Serializable>> existingRecords = new HashSet<>();
+      if(ukFieldNames != null)
+      {
+         QueryInput queryInput = new QueryInput(actionInput.getInstance());
+         queryInput.setSession(actionInput.getSession());
+         queryInput.setTableName(table.getName());
+         queryInput.setTransaction(transaction);
+
+         QQueryFilter filter = new QQueryFilter();
+         if(ukFieldNames.size() == 1)
+         {
+            List<Serializable> values = recordList.stream()
+               .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
+               .map(r -> r.getValue(ukFieldNames.get(0)))
+               .collect(Collectors.toList());
+            filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
+         }
+         else
+         {
+            filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
+            for(QRecord record : recordList)
+            {
+               if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+               {
+                  continue;
+               }
+
+               QQueryFilter subFilter = new QQueryFilter();
+               filter.addSubFilter(subFilter);
+               for(String fieldName : ukFieldNames)
+               {
+                  Serializable value = record.getValue(fieldName);
+                  if(value == null)
+                  {
+                     subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
+                  }
+                  else
+                  {
+                     subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
+                  }
+               }
+            }
+
+            if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - rather - return early. //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               return (existingRecords);
+            }
+         }
+
+         queryInput.setFilter(filter);
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         for(QRecord record : queryOutput.getRecords())
+         {
+            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record);
+            keyValues.ifPresent(existingRecords::add);
+
+         }
+      }
+
+      return (existingRecords);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
+   {
+      try
+      {
+         List<Serializable> keyValues = new ArrayList<>();
+         for(String fieldName : uniqueKey.getFieldNames())
+         {
+            QFieldMetaData field      = table.getField(fieldName);
+            Serializable   value      = record.getValue(fieldName);
+            Serializable   typedValue = ValueUtils.getValueAsFieldType(field.getType(), value);
+            keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue);
+         }
+         return (Optional.of(keyValues));
+      }
+      catch(Exception e)
+      {
+         return (Optional.empty());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** To make a list of unique key values here behave like they do in an RDBMS
+    ** (which is what we're trying to mimic - which is - 2 null values in a field
+    ** aren't considered the same, so they don't violate a unique key) (at least, that's
+    ** how some RDBMS's work, right??) - use this value instead of nulls in the
+    ** output of getKeyValues - where interestingly, this class always returns
+    ** false in it equals method... Unclear how bad this is, e.g., if it's violating
+    ** the contract for equals and hashCode...
+    *******************************************************************************/
+   public static class NullUniqueKeyValue implements Serializable
+   {
+      @Override
+      public boolean equals(Object obj)
+      {
+         return (false);
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java

@@ -36,7 +36,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 public class InsertInput extends AbstractTableActionInput
 {
    private QBackendTransaction transaction;
-   private List<QRecord> records;
+   private List<QRecord>       records;
+
+   private boolean skipUniqueKeyCheck = false;
 
 
 
@@ -80,6 +82,7 @@ public class InsertInput extends AbstractTableActionInput
    }
 
 
+
    /*******************************************************************************
     ** Fluent setter for transaction
     **
@@ -111,4 +114,39 @@ public class InsertInput extends AbstractTableActionInput
    {
       this.records = records;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for skipUniqueKeyCheck
+    **
+    *******************************************************************************/
+   public boolean getSkipUniqueKeyCheck()
+   {
+      return skipUniqueKeyCheck;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for skipUniqueKeyCheck
+    **
+    *******************************************************************************/
+   public void setSkipUniqueKeyCheck(boolean skipUniqueKeyCheck)
+   {
+      this.skipUniqueKeyCheck = skipUniqueKeyCheck;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for skipUniqueKeyCheck
+    **
+    *******************************************************************************/
+   public InsertInput withSkipUniqueKeyCheck(boolean skipUniqueKeyCheck)
+   {
+      this.skipUniqueKeyCheck = skipUniqueKeyCheck;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java

@@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -182,6 +183,12 @@ public class MemoryRecordStore
       QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
       for(QRecord record : input.getRecords())
       {
+         if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            outputRecords.add(record);
+            continue;
+         }
+
          /////////////////////////////////////////////////
          // set the next serial in the record if needed //
          /////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java

@@ -28,24 +28,20 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
-import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
@@ -71,6 +67,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep
    public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       this.table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      runBackendStepOutput.addValue(LoadViaInsertStep.FIELD_SKIP_UNIQUE_KEY_CHECK, true);
    }
 
 
@@ -87,7 +88,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
       List<UniqueKey>                         uniqueKeys   = CollectionUtils.nonNullList(table.getUniqueKeys());
       for(UniqueKey uniqueKey : uniqueKeys)
       {
-         existingKeys.put(uniqueKey, getExistingKeys(runBackendStepInput, uniqueKey));
+         existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(runBackendStepInput, null, table, runBackendStepInput.getRecords(), uniqueKey));
          ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLine(Status.ERROR));
       }
 
@@ -142,8 +143,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
             boolean foundDupe = false;
             for(UniqueKey uniqueKey : uniqueKeys)
             {
-               List<Serializable> keyValues = getKeyValues(uniqueKey, record);
+               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
-               if(existingKeys.get(uniqueKey).contains(keyValues) || keysInThisFile.get(uniqueKey).contains(keyValues))
+               if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get())))
                {
                   ukErrorSummaries.get(uniqueKey).incrementCount();
                   foundDupe = true;
@@ -158,8 +159,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
             {
                for(UniqueKey uniqueKey : uniqueKeys)
                {
-                  List<Serializable> keyValues = getKeyValues(uniqueKey, record);
+                  Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
-                  keysInThisFile.get(uniqueKey).add(keyValues);
+                  keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv));
                }
                okSummary.incrementCount();
                runBackendStepOutput.addRecord(record);
@@ -170,81 +171,6 @@ public class BulkInsertTransformStep extends AbstractTransformStep
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private Set<List<Serializable>> getExistingKeys(RunBackendStepInput runBackendStepInput, UniqueKey uniqueKey) throws QException
-   {
-      List<String>            ukFieldNames    = uniqueKey.getFieldNames();
-      Set<List<Serializable>> existingRecords = new HashSet<>();
-      if(ukFieldNames != null)
-      {
-         QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance());
-         queryInput.setSession(runBackendStepInput.getSession());
-         queryInput.setTableName(runBackendStepInput.getTableName());
-         getTransaction().ifPresent(queryInput::setTransaction);
-
-         QQueryFilter filter = new QQueryFilter();
-         if(ukFieldNames.size() == 1)
-         {
-            List<Serializable> values = runBackendStepInput.getRecords().stream()
-               .map(r -> r.getValue(ukFieldNames.get(0)))
-               .collect(Collectors.toList());
-            filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
-         }
-         else
-         {
-            filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
-            for(QRecord record : runBackendStepInput.getRecords())
-            {
-               QQueryFilter subFilter = new QQueryFilter();
-               filter.addSubFilter(subFilter);
-               for(String fieldName : ukFieldNames)
-               {
-                  subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, List.of(record.getValue(fieldName))));
-               }
-            }
-         }
-
-         queryInput.setFilter(filter);
-         QueryOutput queryOutput = new QueryAction().execute(queryInput);
-         for(QRecord record : queryOutput.getRecords())
-         {
-            List<Serializable> keyValues = getKeyValues(ukFieldNames, record);
-            existingRecords.add(keyValues);
-         }
-      }
-
-      return (existingRecords);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private List<Serializable> getKeyValues(UniqueKey uniqueKey, QRecord record)
-   {
-      return (getKeyValues(uniqueKey.getFieldNames(), record));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private List<Serializable> getKeyValues(List<String> fieldNames, QRecord record)
-   {
-      List<Serializable> keyValues = new ArrayList<>();
-      for(String fieldName : fieldNames)
-      {
-         keyValues.add(record.getValue(fieldName));
-      }
-      return keyValues;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java

@@ -48,7 +48,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public class LoadViaInsertOrUpdateStep extends AbstractLoadStep
 {
-   public static final String FIELD_DESTINATION_TABLE = "destinationTable";
+   public static final String FIELD_DESTINATION_TABLE     = "destinationTable";
+   public static final String FIELD_SKIP_UNIQUE_KEY_CHECK = "skipUniqueKeyCheck";
 
    protected List<QRecord> recordsToInsert = null;
    protected List<QRecord> recordsToUpdate = null;
@@ -83,6 +84,12 @@ public class LoadViaInsertOrUpdateStep extends AbstractLoadStep
          insertInput.setRecords(recordsToInsert);
          getTransaction().ifPresent(insertInput::setTransaction);
          insertInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
+
+         if(runBackendStepInput.getValuePrimitiveBoolean(FIELD_SKIP_UNIQUE_KEY_CHECK))
+         {
+            insertInput.setSkipUniqueKeyCheck(true);
+         }
+
          InsertOutput insertOutput = new InsertAction().execute(insertInput);
          runBackendStepOutput.getRecords().addAll(insertOutput.getRecords());
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java

@@ -38,7 +38,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
  *******************************************************************************/
 public class LoadViaInsertStep extends AbstractLoadStep
 {
-   public static final String FIELD_DESTINATION_TABLE = "destinationTable";
+   public static final String FIELD_DESTINATION_TABLE     = "destinationTable";
+   public static final String FIELD_SKIP_UNIQUE_KEY_CHECK = "skipUniqueKeyCheck";
 
 
 
@@ -55,6 +56,12 @@ public class LoadViaInsertStep extends AbstractLoadStep
       insertInput.setRecords(runBackendStepInput.getRecords());
       getTransaction().ifPresent(insertInput::setTransaction);
       insertInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
+
+      if(runBackendStepInput.getValuePrimitiveBoolean(FIELD_SKIP_UNIQUE_KEY_CHECK))
+      {
+         insertInput.setSkipUniqueKeyCheck(true);
+      }
+
       InsertOutput insertOutput = new InsertAction().execute(insertInput);
       runBackendStepOutput.getRecords().addAll(insertOutput.getRecords());
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java

@@ -38,6 +38,7 @@ import java.util.Calendar;
 import java.util.List;
 import java.util.TimeZone;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 
 
 /*******************************************************************************
@@ -632,4 +633,24 @@ public class ValueUtils
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   public static Serializable getValueAsFieldType(QFieldType type, Serializable value)
+   {
+      return switch(type)
+         {
+            case STRING, TEXT, HTML, PASSWORD -> getValueAsString(value);
+            case INTEGER -> getValueAsInteger(value);
+            case DECIMAL -> getValueAsBigDecimal(value);
+            case BOOLEAN -> getValueAsBoolean(value);
+            case DATE -> getValueAsLocalDate(value);
+            case TIME -> getValueAsLocalTime(value);
+            case DATE_TIME -> getValueAsInstant(value);
+            case BLOB -> getValueAsByteArray(value);
+         };
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java

@@ -0,0 +1,221 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+
+/*******************************************************************************
+ ** Unit test for ChildInserterPostInsertCustomizer
+ **
+ ** We'll use person & shape tables here, w/ the favoriteShapeId foreign key,
+ ** so a rule of "every time we insert a person, if they aren't already pointed at
+ ** a favoriteShape, insert a new shape for them".
+ *******************************************************************************/
+class ChildInserterPostInsertCustomizerTest
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testEmptyCases() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      addPostInsertActionToTable(qInstance);
+
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of());
+
+      ////////////////////////////////////////
+      // just looking for no exception here //
+      ////////////////////////////////////////
+      new InsertAction().execute(insertInput);
+      assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now insert one person, but they shouldn't need a favoriteShape to be inserted - again, make sure we don't blow up and that no shapes get inserted. //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      insertInput.setRecords(List.of(new QRecord().withValue("firstName", "James").withValue("favoriteShapeId", -1)));
+      new InsertAction().execute(insertInput);
+      assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void addPostInsertActionToTable(QInstance qInstance)
+   {
+      qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD.getTableCustomizer(), new QCodeReference(PersonPostInsertAddFavoriteShapeCustomizer.class, QCodeUsage.CUSTOMIZER));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSimpleCase() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      addPostInsertActionToTable(qInstance);
+
+      assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
+
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("firstName", "Darin")
+      ));
+      InsertOutput insertOutput    = new InsertAction().execute(insertInput);
+      Serializable favoriteShapeId = insertOutput.getRecords().get(0).getValue("favoriteShapeId");
+      assertNotNull(favoriteShapeId);
+
+      List<QRecord> shapeRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE);
+      assertEquals(1, shapeRecords.size());
+      assertEquals(favoriteShapeId, shapeRecords.get(0).getValue("id"));
+      assertEquals("Darin's favorite shape!", shapeRecords.get(0).getValue("name"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testComplexCase() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      addPostInsertActionToTable(qInstance);
+
+      assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
+
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("firstName", "Darin"),
+         new QRecord().withValue("firstName", "James").withValue("favoriteShapeId", -1),
+         new QRecord().withValue("firstName", "Tim"),
+         new QRecord().withValue("firstName", "Garret").withValue("favoriteShapeId", -2)
+      ));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+      assertEquals(1, insertOutput.getRecords().get(0).getValue("favoriteShapeId"));
+      assertEquals(-1, insertOutput.getRecords().get(1).getValue("favoriteShapeId"));
+      assertEquals(2, insertOutput.getRecords().get(2).getValue("favoriteShapeId"));
+      assertEquals(-2, insertOutput.getRecords().get(3).getValue("favoriteShapeId"));
+
+      List<QRecord> shapeRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE);
+      assertEquals(2, shapeRecords.size());
+      assertEquals(1, shapeRecords.get(0).getValue("id"));
+      assertEquals(2, shapeRecords.get(1).getValue("id"));
+      assertEquals("Darin's favorite shape!", shapeRecords.get(0).getValue("name"));
+      assertEquals("Tim's favorite shape!", shapeRecords.get(1).getValue("name"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class PersonPostInsertAddFavoriteShapeCustomizer extends ChildInserterPostInsertCustomizer
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public QRecord buildChildForRecord(QRecord parentRecord) throws QException
+      {
+         return (new QRecord().withValue("name", parentRecord.getValue("firstName") + "'s favorite shape!"));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public String getChildTableName()
+      {
+         return (TestUtils.TABLE_NAME_SHAPE);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public String getForeignKeyFieldName()
+      {
+         return ("favoriteShapeId");
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public RelationshipType getRelationshipType()
+      {
+         return (RelationshipType.PARENT_POINTS_AT_CHILD);
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java

@@ -28,9 +28,20 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /*******************************************************************************
@@ -40,6 +51,18 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 class InsertActionTest
 {
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
    /*******************************************************************************
     ** At the core level, there isn't much that can be asserted, as it uses the
     ** mock implementation - just confirming that all of the "wiring" works.
@@ -51,8 +74,8 @@ class InsertActionTest
       InsertInput request = new InsertInput(TestUtils.defineInstance());
       request.setSession(TestUtils.getMockSession());
       request.setTableName("person");
-      List<QRecord> records =new ArrayList<>();
+      List<QRecord> records = new ArrayList<>();
-      QRecord record = new QRecord();
+      QRecord       record  = new QRecord();
       record.setValue("firstName", "James");
       records.add(record);
       request.setRecords(records);
@@ -60,4 +83,134 @@ class InsertActionTest
       assertNotNull(result);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testUniqueKeysPreExisting() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff")
+      ));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+      assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
+
+      ///////////////////////////////////////////////////////
+      // try to insert that person again - shouldn't work. //
+      ///////////////////////////////////////////////////////
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff")
+      ));
+      insertOutput = new InsertAction().execute(insertInput);
+      assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
+      assertNull(insertOutput.getRecords().get(0).getValueInteger("id"));
+      assertEquals(1, insertOutput.getRecords().get(0).getErrors().size());
+      assertThat(insertOutput.getRecords().get(0).getErrors().get(0)).contains("Another record already exists with this First Name and Last Name");
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // try to insert that person again, with 2 others - the 2 should work, but the one fail //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("lastName", "Smith"),
+         new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"),
+         new QRecord().withValue("firstName", "Trevor").withValue("lastName", "Kelkhoff")
+      ));
+      insertOutput = new InsertAction().execute(insertInput);
+      assertEquals(3, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
+      assertNotNull(insertOutput.getRecords().get(0).getValueInteger("id"));
+      assertNull(insertOutput.getRecords().get(1).getValueInteger("id"));
+      assertNotNull(insertOutput.getRecords().get(2).getValueInteger("id"));
+      assertEquals(0, insertOutput.getRecords().get(0).getErrors().size());
+      assertEquals(1, insertOutput.getRecords().get(1).getErrors().size());
+      assertEquals(0, insertOutput.getRecords().get(2).getErrors().size());
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testUniqueKeysWithinBatch() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"),
+         new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff")
+      ));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+      assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
+      assertEquals(1, insertOutput.getRecords().get(0).getValueInteger("id"));
+      assertNull(insertOutput.getRecords().get(1).getValueInteger("id"));
+      assertEquals(1, insertOutput.getRecords().get(1).getErrors().size());
+      assertThat(insertOutput.getRecords().get(1).getErrors().get(0)).contains("Another record already exists with this First Name and Last Name");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSingleColumnUniqueKey() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      qInstance.getTable(TestUtils.TABLE_NAME_SHAPE)
+         .withUniqueKey(new UniqueKey("name"));
+
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("name", "Circle"),
+         new QRecord().withValue("name", "Circle")
+      ));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+      assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
+      assertEquals(1, insertOutput.getRecords().get(0).getValueInteger("id"));
+      assertNull(insertOutput.getRecords().get(1).getValueInteger("id"));
+      assertEquals(1, insertOutput.getRecords().get(1).getErrors().size());
+      assertThat(insertOutput.getRecords().get(1).getErrors().get(0)).contains("Another record already exists with this Name");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSkippingUniqueKeys() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setSkipUniqueKeyCheck(true);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"),
+         new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff")
+      ));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+      assertEquals(2, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
+      assertEquals(1, insertOutput.getRecords().get(0).getValueInteger("id"));
+      assertEquals(2, insertOutput.getRecords().get(1).getValueInteger("id"));
+      assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(1).getErrors()));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -98,7 +98,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule;
 import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
-import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryTableBackendDetails;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
 import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
@@ -652,8 +651,6 @@ public class TestUtils
       return (new QTableMetaData()
          .withName(TABLE_NAME_PERSON_MEMORY_CACHE)
          .withBackendName(MEMORY_BACKEND_NAME)
-         .withBackendDetails(new MemoryTableBackendDetails()
-            .withCloneUponStore(true))
          .withPrimaryKeyField("id")
          .withUniqueKey(uniqueKey)
          .withFields(TestUtils.defineTablePerson().getFields()))

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java

@@ -114,13 +114,25 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
                List<Object>  params      = new ArrayList<>();
                int           recordIndex = 0;
 
+               //////////////////////////////////////////////////////
+               // for each record in the page:                     //
+               // - if it has errors, skip it                      //
+               // - else add a "(?,?,...,?)," clause to the INSERT //
+               // - then add all fields into the params list       //
+               //////////////////////////////////////////////////////
                for(QRecord record : page)
                {
+                  if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+                  {
+                     continue;
+                  }
+
                   if(recordIndex++ > 0)
                   {
                      sql.append(",");
                   }
                   sql.append("(").append(questionMarks).append(")");
+
                   for(QFieldMetaData field : insertableFields)
                   {
                      Serializable value = record.getValue(field.getName());
@@ -129,6 +141,23 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
                   }
                }
 
+               ////////////////////////////////////////////////////////////////////////////////////////
+               // if all records had errors, copy them to the output, and continue w/o running query //
+               ////////////////////////////////////////////////////////////////////////////////////////
+               if(recordIndex == 0)
+               {
+                  for(QRecord record : page)
+                  {
+                     QRecord outputRecord = new QRecord(record);
+                     outputRecords.add(outputRecord);
+                  }
+                  continue;
+               }
+
+               ///////////////////////////////////////////////////////////
+               // execute the insert, then foreach record in the input, //
+               // add it to the output, and set its generated id too.   //
+               ///////////////////////////////////////////////////////////
                // todo sql customization - can edit sql and/or param list
                // todo - non-serial-id style tables
                // todo - other generated values, e.g., createDate...  maybe need to re-select?
@@ -136,10 +165,14 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
                int           index  = 0;
                for(QRecord record : page)
                {
-                  Integer id           = idList.get(index++);
                   QRecord outputRecord = new QRecord(record);
-                  outputRecord.setValue(table.getPrimaryKeyField(), id);
                   outputRecords.add(outputRecord);
+
+                  if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
+                  {
+                     Integer id = idList.get(index++);
+                     outputRecord.setValue(table.getPrimaryKeyField(), id);
+                  }
                }
             }
          }

qqq-dev-tools/CURRENT-SNAPSHOT-VERSION

@@ -1 +1 @@
-0.8.0
+0.9.0

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java

@@ -522,6 +522,11 @@ public class QJavalinImplementation
          InsertAction insertAction = new InsertAction();
          InsertOutput insertOutput = insertAction.execute(insertInput);
 
+         if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors()))
+         {
+            throw (new QUserFacingException("Error inserting " + qInstance.getTable(table).getLabel() + ": " + insertOutput.getRecords().get(0).getErrors().get(0)));
+         }
+
          context.result(JsonUtils.toJson(insertOutput));
       }
       catch(Exception e)