diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryTableBackendDetails.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java similarity index 70% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryTableBackendDetails.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java index c3726fb9..ac7d195f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryTableBackendDetails.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java @@ -19,51 +19,47 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory; +package com.kingsrook.qqq.backend.core.actions.customizers; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* ** *******************************************************************************/ -public class MemoryTableBackendDetails extends QTableBackendDetails +public abstract class AbstractPostInsertCustomizer { - private boolean cloneUponStore = false; + protected InsertInput insertInput; /******************************************************************************* - ** Getter for cloneUponStore ** *******************************************************************************/ - public boolean getCloneUponStore() + public abstract List apply(List records); + + + + /******************************************************************************* + ** Getter for insertInput + ** + *******************************************************************************/ + public InsertInput getInsertInput() { - return cloneUponStore; + return insertInput; } /******************************************************************************* - ** Setter for cloneUponStore + ** Setter for insertInput ** *******************************************************************************/ - public void setCloneUponStore(boolean cloneUponStore) + public void setInsertInput(InsertInput insertInput) { - this.cloneUponStore = cloneUponStore; + this.insertInput = insertInput; } - - - - /******************************************************************************* - ** Fluent setter for cloneUponStore - ** - *******************************************************************************/ - public MemoryTableBackendDetails withCloneUponStore(boolean cloneUponStore) - { - this.cloneUponStore = cloneUponStore; - return (this); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java new file mode 100644 index 00000000..6b731012 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java @@ -0,0 +1,160 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.customizers; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Standard/re-usable post-insert customizer, for the use case where, when we + ** do an insert into table "parent", we want a record automatically inserted into + ** table "child", and there's a foreign key in "parent", pointed at "child" + ** e.g., named: "parent.childId". + ** + ** A similar use-case would have the foreign key in the child table - in which case, + ** we could add a "Type" enum, plus abstract method to get our "Type", then logic + ** to switch behavior based on type. See existing type enum, but w/ only 1 case :) + *******************************************************************************/ +public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer +{ + public enum RelationshipType + { + PARENT_POINTS_AT_CHILD + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract QRecord buildChildForRecord(QRecord parentRecord) throws QException; + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract String getChildTableName(); + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract String getForeignKeyFieldName(); + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract RelationshipType getRelationshipType(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) + { + try + { + List rs = new ArrayList<>(); + List childrenToInsert = new ArrayList<>(); + QTableMetaData table = getInsertInput().getTable(); + QTableMetaData childTable = getInsertInput().getInstance().getTable(getChildTableName()); + + //////////////////////////////////////////////////////////////////////////////// + // iterate over the inserted records, building a list child records to insert // + // for ones missing a value in the foreign key field. // + //////////////////////////////////////////////////////////////////////////////// + for(QRecord record : records) + { + if(record.getValue(getForeignKeyFieldName()) == null) + { + childrenToInsert.add(buildChildForRecord(record)); + } + } + + /////////////////////////////////////////////////////////////////////////////////// + // if there are no children to insert, then just return the original record list // + /////////////////////////////////////////////////////////////////////////////////// + if(childrenToInsert.isEmpty()) + { + return (records); + } + + ///////////////////////// + // insert the children // + ///////////////////////// + InsertInput insertInput = new InsertInput(getInsertInput().getInstance()); + insertInput.setSession(getInsertInput().getSession()); + insertInput.setTableName(getChildTableName()); + insertInput.setRecords(childrenToInsert); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + Iterator insertedRecordIterator = insertOutput.getRecords().iterator(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // iterate over the original list of records again - for any that need a child (e.g., are missing // + // foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + List recordsToUpdate = new ArrayList<>(); + for(QRecord record : records) + { + Serializable primaryKey = record.getValue(table.getPrimaryKeyField()); + if(record.getValue(getForeignKeyFieldName()) == null) + { + Serializable foreignKey = insertedRecordIterator.next().getValue(childTable.getPrimaryKeyField()); + recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey)); + record.setValue(getForeignKeyFieldName(), foreignKey); + rs.add(record); + } + else + { + rs.add(record); + } + } + + //////////////////////////////////////////////////////////////////////////// + // update the originally inserted records to reference their new children // + //////////////////////////////////////////////////////////////////////////// + UpdateInput updateInput = new UpdateInput(insertInput.getInstance()); + updateInput.setSession(getInsertInput().getSession()); + updateInput.setTableName(getInsertInput().getTableName()); + updateInput.setRecords(recordsToUpdate); + new UpdateAction().execute(updateInput); + + return (rs); + } + catch(Exception e) + { + throw new RuntimeException("Error inserting new child records for new parent records", e); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index a0d3ec0a..62c88535 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -61,6 +61,21 @@ public class QCodeLoader + /******************************************************************************* + ** + *******************************************************************************/ + public static Optional getTableCustomizer(Class expectedClass, QTableMetaData table, String customizerName) + { + Optional codeReference = table.getCustomizer(customizerName); + if(codeReference.isPresent()) + { + return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get()))); + } + return (Optional.empty()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java index f61bf8ce..4e8e22b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java @@ -49,6 +49,18 @@ public class TableCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + public TableCustomizer(String role, Class expectedType) + { + this.role = role; + this.expectedType = expectedType; + this.validationFunction = null; + } + + + /******************************************************************************* ** Getter for role ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java index 633901ee..36e467df 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java @@ -47,8 +47,10 @@ public enum TableCustomizers { @SuppressWarnings("unchecked") Function function = (Function) x; - QRecord output = function.apply(new QRecord()); - }))); + QRecord output = function.apply(new QRecord()); + }))), + + POST_INSERT_RECORD(new TableCustomizer("postInsertRecord", AbstractPostInsertCustomizer.class)); private final TableCustomizer tableCustomizer; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index cbca6067..e92481bb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -22,17 +22,32 @@ package com.kingsrook.qqq.backend.core.actions.tables; +import java.io.Serializable; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -54,20 +69,96 @@ public class InsertAction extends AbstractQActionFunction postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole()); setAutomationStatusField(insertInput); - ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), insertInput.getTable(), insertInput.getRecords()); + ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords()); // todo - need to handle records with errors coming out of here... QBackendModuleInterface qModule = getBackendModuleInterface(insertInput); // todo pre-customization - just get to modify the request? + + setErrorsIfUniqueKeyErrors(insertInput, table); + InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput); // todo post-customization - can do whatever w/ the result if you want + + if(postInsertCustomizer.isPresent()) + { + postInsertCustomizer.get().setInsertInput(insertInput); + insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords())); + } + return insertOutput; } + /******************************************************************************* + ** + *******************************************************************************/ + private void setErrorsIfUniqueKeyErrors(InsertInput insertInput, QTableMetaData table) throws QException + { + if(CollectionUtils.nullSafeHasContents(table.getUniqueKeys())) + { + Map>> keysInThisList = new HashMap<>(); + if(insertInput.getSkipUniqueKeyCheck()) + { + LOG.debug("Skipping unique key check in " + insertInput.getTableName() + " insert."); + return; + } + + //////////////////////////////////////////// + // check for any pre-existing unique keys // + //////////////////////////////////////////// + Map>> existingKeys = new HashMap<>(); + List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); + for(UniqueKey uniqueKey : uniqueKeys) + { + existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(insertInput, insertInput.getTransaction(), table, insertInput.getRecords(), uniqueKey)); + } + + ///////////////////////////////////// + // make sure this map is populated // + ///////////////////////////////////// + uniqueKeys.forEach(uk -> keysInThisList.computeIfAbsent(uk, x -> new HashSet<>())); + + for(QRecord record : insertInput.getRecords()) + { + ////////////////////////////////////////////////////////// + // check if this record violates any of the unique keys // + ////////////////////////////////////////////////////////// + boolean foundDupe = false; + for(UniqueKey uniqueKey : uniqueKeys) + { + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get()))) + { + record.addError("Another record already exists with this " + uniqueKey.getDescription(table)); + foundDupe = true; + break; + } + } + + /////////////////////////////////////////////////////////////////////////////// + // if this record doesn't violate any uk's, then we can add it to the output // + /////////////////////////////////////////////////////////////////////////////// + if(!foundDupe) + { + for(UniqueKey uniqueKey : uniqueKeys) + { + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + keyValues.ifPresent(kv -> keysInThisList.get(uniqueKey).add(kv)); + } + } + } + } + } + + + /******************************************************************************* ** If the table being inserted into uses an automation-status field, populate it now. *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java new file mode 100644 index 00000000..15e2ca03 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java @@ -0,0 +1,171 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.tables.helpers; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Methods to help with unique key checks. + *******************************************************************************/ +public class UniqueKeyHelper +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static Set> getExistingKeys(AbstractActionInput actionInput, QBackendTransaction transaction, QTableMetaData table, List recordList, UniqueKey uniqueKey) throws QException + { + List ukFieldNames = uniqueKey.getFieldNames(); + Set> existingRecords = new HashSet<>(); + if(ukFieldNames != null) + { + QueryInput queryInput = new QueryInput(actionInput.getInstance()); + queryInput.setSession(actionInput.getSession()); + queryInput.setTableName(table.getName()); + queryInput.setTransaction(transaction); + + QQueryFilter filter = new QQueryFilter(); + if(ukFieldNames.size() == 1) + { + List values = recordList.stream() + .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())) + .map(r -> r.getValue(ukFieldNames.get(0))) + .collect(Collectors.toList()); + filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values)); + } + else + { + filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR); + for(QRecord record : recordList) + { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + continue; + } + + QQueryFilter subFilter = new QQueryFilter(); + filter.addSubFilter(subFilter); + for(String fieldName : ukFieldNames) + { + Serializable value = record.getValue(fieldName); + if(value == null) + { + subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK)); + } + else + { + subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value)); + } + } + } + + if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters())) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - rather - return early. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (existingRecords); + } + } + + queryInput.setFilter(filter); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + for(QRecord record : queryOutput.getRecords()) + { + Optional> keyValues = getKeyValues(table, uniqueKey, record); + keyValues.ifPresent(existingRecords::add); + + } + } + + return (existingRecords); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Optional> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record) + { + try + { + List keyValues = new ArrayList<>(); + for(String fieldName : uniqueKey.getFieldNames()) + { + QFieldMetaData field = table.getField(fieldName); + Serializable value = record.getValue(fieldName); + Serializable typedValue = ValueUtils.getValueAsFieldType(field.getType(), value); + keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue); + } + return (Optional.of(keyValues)); + } + catch(Exception e) + { + return (Optional.empty()); + } + } + + + + /******************************************************************************* + ** To make a list of unique key values here behave like they do in an RDBMS + ** (which is what we're trying to mimic - which is - 2 null values in a field + ** aren't considered the same, so they don't violate a unique key) (at least, that's + ** how some RDBMS's work, right??) - use this value instead of nulls in the + ** output of getKeyValues - where interestingly, this class always returns + ** false in it equals method... Unclear how bad this is, e.g., if it's violating + ** the contract for equals and hashCode... + *******************************************************************************/ + public static class NullUniqueKeyValue implements Serializable + { + @Override + public boolean equals(Object obj) + { + return (false); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java index 24320d98..074de4cf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java @@ -36,7 +36,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; public class InsertInput extends AbstractTableActionInput { private QBackendTransaction transaction; - private List records; + private List records; + + private boolean skipUniqueKeyCheck = false; @@ -80,6 +82,7 @@ public class InsertInput extends AbstractTableActionInput } + /******************************************************************************* ** Fluent setter for transaction ** @@ -111,4 +114,39 @@ public class InsertInput extends AbstractTableActionInput { this.records = records; } + + + + /******************************************************************************* + ** Getter for skipUniqueKeyCheck + ** + *******************************************************************************/ + public boolean getSkipUniqueKeyCheck() + { + return skipUniqueKeyCheck; + } + + + + /******************************************************************************* + ** Setter for skipUniqueKeyCheck + ** + *******************************************************************************/ + public void setSkipUniqueKeyCheck(boolean skipUniqueKeyCheck) + { + this.skipUniqueKeyCheck = skipUniqueKeyCheck; + } + + + + /******************************************************************************* + ** Fluent setter for skipUniqueKeyCheck + ** + *******************************************************************************/ + public InsertInput withSkipUniqueKeyCheck(boolean skipUniqueKeyCheck) + { + this.skipUniqueKeyCheck = skipUniqueKeyCheck; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index bc3702dd..255b481b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* @@ -182,6 +183,12 @@ public class MemoryRecordStore QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField()); for(QRecord record : input.getRecords()) { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + outputRecords.add(record); + continue; + } + ///////////////////////////////////////////////// // set the next serial in the record if needed // ///////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 0f9c12ad..97e182de 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -28,24 +28,20 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; -import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.Status; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -71,6 +67,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { this.table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue(LoadViaInsertStep.FIELD_SKIP_UNIQUE_KEY_CHECK, true); } @@ -87,7 +88,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); for(UniqueKey uniqueKey : uniqueKeys) { - existingKeys.put(uniqueKey, getExistingKeys(runBackendStepInput, uniqueKey)); + existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(runBackendStepInput, null, table, runBackendStepInput.getRecords(), uniqueKey)); ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLine(Status.ERROR)); } @@ -142,8 +143,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep boolean foundDupe = false; for(UniqueKey uniqueKey : uniqueKeys) { - List keyValues = getKeyValues(uniqueKey, record); - if(existingKeys.get(uniqueKey).contains(keyValues) || keysInThisFile.get(uniqueKey).contains(keyValues)) + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get()))) { ukErrorSummaries.get(uniqueKey).incrementCount(); foundDupe = true; @@ -158,8 +159,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep { for(UniqueKey uniqueKey : uniqueKeys) { - List keyValues = getKeyValues(uniqueKey, record); - keysInThisFile.get(uniqueKey).add(keyValues); + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv)); } okSummary.incrementCount(); runBackendStepOutput.addRecord(record); @@ -170,81 +171,6 @@ public class BulkInsertTransformStep extends AbstractTransformStep - /******************************************************************************* - ** - *******************************************************************************/ - private Set> getExistingKeys(RunBackendStepInput runBackendStepInput, UniqueKey uniqueKey) throws QException - { - List ukFieldNames = uniqueKey.getFieldNames(); - Set> existingRecords = new HashSet<>(); - if(ukFieldNames != null) - { - QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance()); - queryInput.setSession(runBackendStepInput.getSession()); - queryInput.setTableName(runBackendStepInput.getTableName()); - getTransaction().ifPresent(queryInput::setTransaction); - - QQueryFilter filter = new QQueryFilter(); - if(ukFieldNames.size() == 1) - { - List values = runBackendStepInput.getRecords().stream() - .map(r -> r.getValue(ukFieldNames.get(0))) - .collect(Collectors.toList()); - filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values)); - } - else - { - filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR); - for(QRecord record : runBackendStepInput.getRecords()) - { - QQueryFilter subFilter = new QQueryFilter(); - filter.addSubFilter(subFilter); - for(String fieldName : ukFieldNames) - { - subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, List.of(record.getValue(fieldName)))); - } - } - } - - queryInput.setFilter(filter); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - for(QRecord record : queryOutput.getRecords()) - { - List keyValues = getKeyValues(ukFieldNames, record); - existingRecords.add(keyValues); - } - } - - return (existingRecords); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private List getKeyValues(UniqueKey uniqueKey, QRecord record) - { - return (getKeyValues(uniqueKey.getFieldNames(), record)); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private List getKeyValues(List fieldNames, QRecord record) - { - List keyValues = new ArrayList<>(); - for(String fieldName : fieldNames) - { - keyValues.add(record.getValue(fieldName)); - } - return keyValues; - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java index eb6dff48..875c2fb7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java @@ -48,7 +48,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; *******************************************************************************/ public class LoadViaInsertOrUpdateStep extends AbstractLoadStep { - public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_SKIP_UNIQUE_KEY_CHECK = "skipUniqueKeyCheck"; protected List recordsToInsert = null; protected List recordsToUpdate = null; @@ -83,6 +84,12 @@ public class LoadViaInsertOrUpdateStep extends AbstractLoadStep insertInput.setRecords(recordsToInsert); getTransaction().ifPresent(insertInput::setTransaction); insertInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); + + if(runBackendStepInput.getValuePrimitiveBoolean(FIELD_SKIP_UNIQUE_KEY_CHECK)) + { + insertInput.setSkipUniqueKeyCheck(true); + } + InsertOutput insertOutput = new InsertAction().execute(insertInput); runBackendStepOutput.getRecords().addAll(insertOutput.getRecords()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java index 8e6a7979..8c3ea7d5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java @@ -38,7 +38,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; *******************************************************************************/ public class LoadViaInsertStep extends AbstractLoadStep { - public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_SKIP_UNIQUE_KEY_CHECK = "skipUniqueKeyCheck"; @@ -55,6 +56,12 @@ public class LoadViaInsertStep extends AbstractLoadStep insertInput.setRecords(runBackendStepInput.getRecords()); getTransaction().ifPresent(insertInput::setTransaction); insertInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); + + if(runBackendStepInput.getValuePrimitiveBoolean(FIELD_SKIP_UNIQUE_KEY_CHECK)) + { + insertInput.setSkipUniqueKeyCheck(true); + } + InsertOutput insertOutput = new InsertAction().execute(insertInput); runBackendStepOutput.getRecords().addAll(insertOutput.getRecords()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index 52fb127c..3fa4d1f1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -38,6 +38,7 @@ import java.util.Calendar; import java.util.List; import java.util.TimeZone; import com.kingsrook.qqq.backend.core.exceptions.QValueException; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; /******************************************************************************* @@ -632,4 +633,24 @@ public class ValueUtils } } + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("checkstyle:indentation") + public static Serializable getValueAsFieldType(QFieldType type, Serializable value) + { + return switch(type) + { + case STRING, TEXT, HTML, PASSWORD -> getValueAsString(value); + case INTEGER -> getValueAsInteger(value); + case DECIMAL -> getValueAsBigDecimal(value); + case BOOLEAN -> getValueAsBoolean(value); + case DATE -> getValueAsLocalDate(value); + case TIME -> getValueAsLocalTime(value); + case DATE_TIME -> getValueAsInstant(value); + case BLOB -> getValueAsByteArray(value); + }; + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java new file mode 100644 index 00000000..20335285 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java @@ -0,0 +1,221 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.customizers; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for ChildInserterPostInsertCustomizer + ** + ** We'll use person & shape tables here, w/ the favoriteShapeId foreign key, + ** so a rule of "every time we insert a person, if they aren't already pointed at + ** a favoriteShape, insert a new shape for them". + *******************************************************************************/ +class ChildInserterPostInsertCustomizerTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testEmptyCases() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + addPostInsertActionToTable(qInstance); + + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of()); + + //////////////////////////////////////// + // just looking for no exception here // + //////////////////////////////////////// + new InsertAction().execute(insertInput); + assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size()); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now insert one person, but they shouldn't need a favoriteShape to be inserted - again, make sure we don't blow up and that no shapes get inserted. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + insertInput.setRecords(List.of(new QRecord().withValue("firstName", "James").withValue("favoriteShapeId", -1))); + new InsertAction().execute(insertInput); + assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void addPostInsertActionToTable(QInstance qInstance) + { + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withCustomizer(TableCustomizers.POST_INSERT_RECORD.getTableCustomizer(), new QCodeReference(PersonPostInsertAddFavoriteShapeCustomizer.class, QCodeUsage.CUSTOMIZER)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleCase() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + addPostInsertActionToTable(qInstance); + + assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size()); + + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "Darin") + )); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + Serializable favoriteShapeId = insertOutput.getRecords().get(0).getValue("favoriteShapeId"); + assertNotNull(favoriteShapeId); + + List shapeRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE); + assertEquals(1, shapeRecords.size()); + assertEquals(favoriteShapeId, shapeRecords.get(0).getValue("id")); + assertEquals("Darin's favorite shape!", shapeRecords.get(0).getValue("name")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testComplexCase() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + addPostInsertActionToTable(qInstance); + + assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size()); + + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "Darin"), + new QRecord().withValue("firstName", "James").withValue("favoriteShapeId", -1), + new QRecord().withValue("firstName", "Tim"), + new QRecord().withValue("firstName", "Garret").withValue("favoriteShapeId", -2) + )); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(1, insertOutput.getRecords().get(0).getValue("favoriteShapeId")); + assertEquals(-1, insertOutput.getRecords().get(1).getValue("favoriteShapeId")); + assertEquals(2, insertOutput.getRecords().get(2).getValue("favoriteShapeId")); + assertEquals(-2, insertOutput.getRecords().get(3).getValue("favoriteShapeId")); + + List shapeRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE); + assertEquals(2, shapeRecords.size()); + assertEquals(1, shapeRecords.get(0).getValue("id")); + assertEquals(2, shapeRecords.get(1).getValue("id")); + assertEquals("Darin's favorite shape!", shapeRecords.get(0).getValue("name")); + assertEquals("Tim's favorite shape!", shapeRecords.get(1).getValue("name")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class PersonPostInsertAddFavoriteShapeCustomizer extends ChildInserterPostInsertCustomizer + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QRecord buildChildForRecord(QRecord parentRecord) throws QException + { + return (new QRecord().withValue("name", parentRecord.getValue("firstName") + "'s favorite shape!")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getChildTableName() + { + return (TestUtils.TABLE_NAME_SHAPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getForeignKeyFieldName() + { + return ("favoriteShapeId"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RelationshipType getRelationshipType() + { + return (RelationshipType.PARENT_POINTS_AT_CHILD); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java index 08ac5ef0..f756045a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java @@ -28,9 +28,20 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -40,6 +51,18 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; class InsertActionTest { + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + /******************************************************************************* ** At the core level, there isn't much that can be asserted, as it uses the ** mock implementation - just confirming that all of the "wiring" works. @@ -51,8 +74,8 @@ class InsertActionTest InsertInput request = new InsertInput(TestUtils.defineInstance()); request.setSession(TestUtils.getMockSession()); request.setTableName("person"); - List records =new ArrayList<>(); - QRecord record = new QRecord(); + List records = new ArrayList<>(); + QRecord record = new QRecord(); record.setValue("firstName", "James"); records.add(record); request.setRecords(records); @@ -60,4 +83,134 @@ class InsertActionTest assertNotNull(result); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUniqueKeysPreExisting() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff") + )); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size()); + + /////////////////////////////////////////////////////// + // try to insert that person again - shouldn't work. // + /////////////////////////////////////////////////////// + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff") + )); + insertOutput = new InsertAction().execute(insertInput); + assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size()); + assertNull(insertOutput.getRecords().get(0).getValueInteger("id")); + assertEquals(1, insertOutput.getRecords().get(0).getErrors().size()); + assertThat(insertOutput.getRecords().get(0).getErrors().get(0)).contains("Another record already exists with this First Name and Last Name"); + + ////////////////////////////////////////////////////////////////////////////////////////// + // try to insert that person again, with 2 others - the 2 should work, but the one fail // + ////////////////////////////////////////////////////////////////////////////////////////// + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "Darin").withValue("lastName", "Smith"), + new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"), + new QRecord().withValue("firstName", "Trevor").withValue("lastName", "Kelkhoff") + )); + insertOutput = new InsertAction().execute(insertInput); + assertEquals(3, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size()); + assertNotNull(insertOutput.getRecords().get(0).getValueInteger("id")); + assertNull(insertOutput.getRecords().get(1).getValueInteger("id")); + assertNotNull(insertOutput.getRecords().get(2).getValueInteger("id")); + assertEquals(0, insertOutput.getRecords().get(0).getErrors().size()); + assertEquals(1, insertOutput.getRecords().get(1).getErrors().size()); + assertEquals(0, insertOutput.getRecords().get(2).getErrors().size()); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUniqueKeysWithinBatch() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"), + new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff") + )); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + + assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size()); + assertEquals(1, insertOutput.getRecords().get(0).getValueInteger("id")); + assertNull(insertOutput.getRecords().get(1).getValueInteger("id")); + assertEquals(1, insertOutput.getRecords().get(1).getErrors().size()); + assertThat(insertOutput.getRecords().get(1).getErrors().get(0)).contains("Another record already exists with this First Name and Last Name"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSingleColumnUniqueKey() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable(TestUtils.TABLE_NAME_SHAPE) + .withUniqueKey(new UniqueKey("name")); + + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE); + insertInput.setRecords(List.of( + new QRecord().withValue("name", "Circle"), + new QRecord().withValue("name", "Circle") + )); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + + assertEquals(1, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size()); + assertEquals(1, insertOutput.getRecords().get(0).getValueInteger("id")); + assertNull(insertOutput.getRecords().get(1).getValueInteger("id")); + assertEquals(1, insertOutput.getRecords().get(1).getErrors().size()); + assertThat(insertOutput.getRecords().get(1).getErrors().get(0)).contains("Another record already exists with this Name"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSkippingUniqueKeys() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setSkipUniqueKeyCheck(true); + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"), + new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff") + )); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + + assertEquals(2, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size()); + assertEquals(1, insertOutput.getRecords().get(0).getValueInteger("id")); + assertEquals(2, insertOutput.getRecords().get(1).getValueInteger("id")); + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(1).getErrors())); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 55fe4ace..2a7db9bb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -98,7 +98,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; -import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryTableBackendDetails; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; @@ -652,8 +651,6 @@ public class TestUtils return (new QTableMetaData() .withName(TABLE_NAME_PERSON_MEMORY_CACHE) .withBackendName(MEMORY_BACKEND_NAME) - .withBackendDetails(new MemoryTableBackendDetails() - .withCloneUponStore(true)) .withPrimaryKeyField("id") .withUniqueKey(uniqueKey) .withFields(TestUtils.defineTablePerson().getFields())) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index aa5fe2a4..b8dcb203 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -114,13 +114,25 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte List params = new ArrayList<>(); int recordIndex = 0; + ////////////////////////////////////////////////////// + // for each record in the page: // + // - if it has errors, skip it // + // - else add a "(?,?,...,?)," clause to the INSERT // + // - then add all fields into the params list // + ////////////////////////////////////////////////////// for(QRecord record : page) { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + continue; + } + if(recordIndex++ > 0) { sql.append(","); } sql.append("(").append(questionMarks).append(")"); + for(QFieldMetaData field : insertableFields) { Serializable value = record.getValue(field.getName()); @@ -129,6 +141,23 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte } } + //////////////////////////////////////////////////////////////////////////////////////// + // if all records had errors, copy them to the output, and continue w/o running query // + //////////////////////////////////////////////////////////////////////////////////////// + if(recordIndex == 0) + { + for(QRecord record : page) + { + QRecord outputRecord = new QRecord(record); + outputRecords.add(outputRecord); + } + continue; + } + + /////////////////////////////////////////////////////////// + // execute the insert, then foreach record in the input, // + // add it to the output, and set its generated id too. // + /////////////////////////////////////////////////////////// // todo sql customization - can edit sql and/or param list // todo - non-serial-id style tables // todo - other generated values, e.g., createDate... maybe need to re-select? @@ -136,10 +165,14 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte int index = 0; for(QRecord record : page) { - Integer id = idList.get(index++); QRecord outputRecord = new QRecord(record); - outputRecord.setValue(table.getPrimaryKeyField(), id); outputRecords.add(outputRecord); + + if(CollectionUtils.nullSafeIsEmpty(record.getErrors())) + { + Integer id = idList.get(index++); + outputRecord.setValue(table.getPrimaryKeyField(), id); + } } } } diff --git a/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION b/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION index a3df0a69..ac39a106 100644 --- a/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION +++ b/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION @@ -1 +1 @@ -0.8.0 +0.9.0 diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index b1c1ae26..fb1c32dc 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -522,6 +522,11 @@ public class QJavalinImplementation InsertAction insertAction = new InsertAction(); InsertOutput insertOutput = insertAction.execute(insertInput); + if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors())) + { + throw (new QUserFacingException("Error inserting " + qInstance.getTable(table).getLabel() + ": " + insertOutput.getRecords().get(0).getErrors().get(0))); + } + context.result(JsonUtils.toJson(insertOutput)); } catch(Exception e)