diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 31c3f315..72730b05 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -414,6 +414,9 @@ public class QPossibleValueTranslator QueryOutput queryOutput = new QueryAction().execute(queryInput); + /////////////////////////////////////////////////////////////////////////////////// + // for all records that were found, put a formatted value into cache foreach PVS // + /////////////////////////////////////////////////////////////////////////////////// for(QRecord record : queryOutput.getRecords()) { Serializable pkeyValue = record.getValue(primaryKeyField); @@ -423,6 +426,20 @@ public class QPossibleValueTranslator possibleValueCache.get(possibleValueSource.getName()).put(pkeyValue, formatPossibleValue(possibleValueSource, possibleValue)); } } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // for all pkeys that were NOT found, put a null value into cache foreach PVS (to avoid re-looking up) // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + for(Serializable pkey : page) + { + for(QPossibleValueSource possibleValueSource : possibleValueSources) + { + if(!possibleValueCache.get(possibleValueSource.getName()).containsKey(pkey)) + { + possibleValueCache.get(possibleValueSource.getName()).put(pkey, null); + } + } + } } } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 88bf8272..59e65887 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -36,11 +36,13 @@ import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; @@ -55,6 +57,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; +import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -222,24 +225,34 @@ public class QInstanceValidator *******************************************************************************/ private void validateTables(QInstance qInstance) { - if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()), - "At least 1 table must be defined.")) + if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()), "At least 1 table must be defined.")) { qInstance.getTables().forEach((tableName, table) -> { assertCondition(Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + "."); - validateAppChildHasValidParentAppName(qInstance, table); //////////////////////////////////////// // validate the backend for the table // //////////////////////////////////////// - if(assertCondition(StringUtils.hasContent(table.getBackendName()), - "Missing backend name for table " + tableName + ".")) + if(assertCondition(StringUtils.hasContent(table.getBackendName()), "Missing backend name for table " + tableName + ".")) { if(CollectionUtils.nullSafeHasContents(qInstance.getBackends())) { - assertCondition(qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + "."); + QBackendMetaData backendForTable = qInstance.getBackendForTable(tableName); + if(assertCondition(backendForTable != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + ".")) + { + //////////////////////////////////////////////////////////// + // if the backend requires primary keys, then validate it // + //////////////////////////////////////////////////////////// + if(backendForTable.requiresPrimaryKeyOnTables()) + { + if(assertCondition(StringUtils.hasContent(table.getPrimaryKeyField()), "Missing primary key for table: " + tableName)) + { + assertNoException(() -> table.getField(table.getPrimaryKeyField()), "Primary key for table " + tableName + " is not a recognized field on this table."); + } + } + } } } @@ -329,12 +342,49 @@ public class QInstanceValidator { validateTableUniqueKeys(table); } + + ///////////////////////////////////////////// + // validate the table's associated scripts // + ///////////////////////////////////////////// + if(table.getAssociatedScripts() != null) + { + validateAssociatedScripts(table); + } }); } } + /******************************************************************************* + ** + *******************************************************************************/ + private void validateAssociatedScripts(QTableMetaData table) + { + Set usedFieldNames = new HashSet<>(); + for(AssociatedScript associatedScript : table.getAssociatedScripts()) + { + if(assertCondition(StringUtils.hasContent(associatedScript.getFieldName()), "Table " + table.getName() + " has an associatedScript without a fieldName")) + { + assertCondition(!usedFieldNames.contains(associatedScript.getFieldName()), "Table " + table.getName() + " has more than one associatedScript specifying field: " + associatedScript.getFieldName()); + usedFieldNames.add(associatedScript.getFieldName()); + assertNoException(() -> table.getField(associatedScript.getFieldName()), "Table " + table.getName() + " has an associatedScript specifying an unrecognized field: " + associatedScript.getFieldName()); + } + + assertCondition(associatedScript.getScriptTypeId() != null, "Table " + table.getName() + " associatedScript on field " + associatedScript.getFieldName() + " is missing a scriptTypeId"); + if(associatedScript.getScriptTester() != null) + { + String prefix = "Table " + table.getName() + " associatedScript on field " + associatedScript.getFieldName(); + if(preAssertionsForCodeReference(associatedScript.getScriptTester(), prefix)) + { + validateSimpleCodeReference(prefix, associatedScript.getScriptTester(), TestScriptActionInterface.class); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 0ffbd612..ec37ec0a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -106,6 +106,7 @@ public class RunBackendStepInput extends AbstractActionInput target.setTableName(getTableName()); target.setProcessName(getProcessName()); target.setAsyncJobCallback(getAsyncJobCallback()); + target.setFrontendStepBehavior(getFrontendStepBehavior()); target.setValues(getValues()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java index 28b6198c..bb8bf551 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java @@ -35,6 +35,8 @@ import java.util.Locale; import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.utils.ListingHash; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -42,6 +44,8 @@ import com.kingsrook.qqq.backend.core.utils.ListingHash; *******************************************************************************/ public interface QRecordEnum { + Logger LOG = LogManager.getLogger(QRecordEnum.class); + ListingHash, QRecordEntityField> fieldMapping = new ListingHash<>(); @@ -140,9 +144,9 @@ public interface QRecordEnum } else { - if(!method.getName().equals("getClass")) + if(!method.getName().equals("getClass") && !method.getName().equals("getDeclaringClass") && !method.getName().equals("getPossibleValueId")) { - System.err.println("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported."); + LOG.debug("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported."); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index 6c5d6985..7b108d3a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -60,6 +60,16 @@ public class QBackendMetaData + /******************************************************************************* + ** + *******************************************************************************/ + public boolean requiresPrimaryKeyOnTables() + { + return (true); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java index 67650182..7beb1ec2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration; +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -68,4 +69,15 @@ public class EnumerationBackendModule implements QBackendModuleInterface return new EnumerationQueryAction(); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new EnumerationCountAction(); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountAction.java new file mode 100644 index 00000000..42bd015e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountAction.java @@ -0,0 +1,57 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class EnumerationCountAction implements CountInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public CountOutput execute(CountInput countInput) throws QException + { + QueryInput queryInput = new QueryInput(countInput.getInstance()); + queryInput.setSession(countInput.getSession()); + queryInput.setTableName(countInput.getTableName()); + queryInput.setFilter(countInput.getFilter()); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + CountOutput countOutput = new CountOutput(); + countOutput.setCount(queryOutput.getRecords().size()); + return (countOutput); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java index aa2bdfb1..949c5ff2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java @@ -273,6 +273,14 @@ public class BackendQueryFilterUtils *******************************************************************************/ private static boolean testIn(QFilterCriteria criterion, Serializable value) { + if(CollectionUtils.nullSafeHasContents(criterion.getValues())) + { + if(criterion.getValues().get(0) instanceof String && value instanceof Number) + { + value = String.valueOf(value); + } + } + if(!criterion.getValues().contains(value)) { return (false); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index ebbd80c9..3f96d373 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -373,5 +373,21 @@ public class StreamedETLWithFrontendProcess return (this); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withPreviewStepInputFields(List fieldList) + { + QBackendStepMetaData previewStep = processMetaData.getBackendStep(STEP_NAME_PREVIEW); + for(QFieldMetaData field : fieldList) + { + previewStep.getInputMetaData().withField(field); + } + + return (this); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/StandardProcessSummaryLineProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/StandardProcessSummaryLineProducer.java new file mode 100644 index 00000000..d58ae5ba --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/StandardProcessSummaryLineProducer.java @@ -0,0 +1,80 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.general; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import static com.kingsrook.qqq.backend.core.model.actions.processes.Status.OK; + + +/******************************************************************************* + ** Helper for working with process summary lines + *******************************************************************************/ +public class StandardProcessSummaryLineProducer +{ + + /******************************************************************************* + ** Make a line that'll say " {will be/was/were} inserted" + *******************************************************************************/ + public static ProcessSummaryLine getOkToInsertLine() + { + return new ProcessSummaryLine(OK) + .withMessageSuffix(" inserted") + .withSingularFutureMessage("will be") + .withPluralFutureMessage("will be") + .withSingularPastMessage("was") + .withPluralPastMessage("were"); + } + + + + /******************************************************************************* + ** Make a line that'll say " {will be/was/were} updated" + *******************************************************************************/ + public static ProcessSummaryLine getOkToUpdateLine() + { + return new ProcessSummaryLine(OK) + .withMessageSuffix(" updated") + .withSingularFutureMessage("will be") + .withPluralFutureMessage("will be") + .withSingularPastMessage("was") + .withPluralPastMessage("were"); + } + + + + /******************************************************************************* + ** one-liner for implementing getProcessSummary - just pass your lines in as varargs as in: + ** return (StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate)); + *******************************************************************************/ + public static ArrayList toArrayList(ProcessSummaryLine... lines) + { + ArrayList rs = new ArrayList<>(); + for(ProcessSummaryLine line : lines) + { + line.addSelfToListIfAnyCount(rs); + } + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java new file mode 100644 index 00000000..335956c3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -0,0 +1,230 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.tablesync; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer; +import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** This class is for transforming records from a Source table to a Destination table. + ** + ** The Source table has a (unique/primary) key field: sourceTableKeyField, + ** Which is matched against the Destination table's foreign-key: destinationTableForeignKeyField + *******************************************************************************/ +public abstract class AbstractTableSyncTransformStep extends AbstractTransformStep +{ + private ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine(); + private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine(); + private ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR) + .withMessageSuffix("missing a value for the key field.") + .withSingularFutureMessage("will not be synced, because it is ") + .withPluralFutureMessage("will not be synced, because they are ") + .withSingularPastMessage("was not synced, because it is ") + .withPluralPastMessage("were not synced, because they are "); + + private RunBackendStepInput runBackendStepInput = null; + + private QPossibleValueTranslator possibleValueTranslator; + + private Map> tableMaps = new HashMap<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected QRecord getRecord(String tableName, String fieldName, Serializable value) throws QException + { + if(!tableMaps.containsKey(tableName)) + { + Map recordMap = GeneralProcessUtils.loadTableToMap(runBackendStepInput, tableName, fieldName); + tableMaps.put(tableName, recordMap); + } + + return (tableMaps.get(tableName).get(value)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected Serializable getRecordField(String tableName, String fieldName, Serializable value, String outputField) throws QException + { + QRecord record = getRecord(tableName, fieldName, value); + if(record == null) + { + return (null); + } + + return (record.getValue(outputField)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + return (StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords())) + { + return; + } + + this.runBackendStepInput = runBackendStepInput; + String sourceTableKeyField = runBackendStepInput.getValueString(TableSyncProcess.FIELD_SOURCE_TABLE_KEY_FIELD); + String destinationTableForeignKeyField = runBackendStepInput.getValueString(TableSyncProcess.FIELD_DESTINATION_TABLE_FOREIGN_KEY); + String destinationTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE); + + ////////////////////////////////////// + // extract keys from source records // + ////////////////////////////////////// + List sourceKeyList = runBackendStepInput.getRecords().stream() + .map(r -> r.getValueString(sourceTableKeyField)) + .filter(Objects::nonNull) + .filter(v -> !"".equals(v)) + .collect(Collectors.toList()); + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // query to see if we already have those records in the destination (to determine insert/update) // + /////////////////////////////////////////////////////////////////////////////////////////////////// + Map existingRecordsByForeignKey = Collections.emptyMap(); + if(!sourceKeyList.isEmpty()) + { + QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance()); + queryInput.setSession(runBackendStepInput.getSession()); + queryInput.setTableName(destinationTableName); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList)) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + existingRecordsByForeignKey = CollectionUtils.recordsToMap(queryOutput.getRecords(), destinationTableForeignKeyField); + } + + ///////////////////////////////////////////////////////////////// + // foreach source record, build the record we'll insert/update // + ///////////////////////////////////////////////////////////////// + for(QRecord sourceRecord : runBackendStepInput.getRecords()) + { + Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField); + QRecord existingRecord = existingRecordsByForeignKey.get(sourceKeyValue); + + if(sourceKeyValue == null || "".equals(sourceKeyValue)) + { + errorMissingKeyField.incrementCount(); + + try + { + errorMissingKeyField.setMessageSuffix("missing a value for the field " + runBackendStepInput.getTable().getField(sourceTableKeyField).getLabel()); + } + catch(Exception e) + { + ///////////////////////////////////////// + // just leave the default error suffix // + ///////////////////////////////////////// + } + + continue; + } + + QRecord recordToStore; + if(existingRecord != null) + { + recordToStore = existingRecord; + okToUpdate.incrementCount(); + } + else + { + recordToStore = new QRecord(); + okToInsert.incrementCount(); + } + + recordToStore = populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord); + runBackendStepOutput.addRecord(recordToStore); + } + + //////////////////////////////////////////////// + // populate possible-values for review screen // + //////////////////////////////////////////////// + if(RunProcessInput.FrontendStepBehavior.BREAK.equals(runBackendStepInput.getFrontendStepBehavior())) + { + if(CollectionUtils.nullSafeHasContents(runBackendStepOutput.getRecords())) + { + if(possibleValueTranslator == null) + { + possibleValueTranslator = new QPossibleValueTranslator(runBackendStepInput.getInstance(), runBackendStepInput.getSession()); + } + + possibleValueTranslator.translatePossibleValuesInRecords(runBackendStepInput.getInstance().getTable(destinationTableName), runBackendStepOutput.getRecords()); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java new file mode 100644 index 00000000..35c6224a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.tablesync; + + +import java.util.Collections; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertOrUpdateStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + + +/******************************************************************************* + ** Definition for Standard process to sync data from one table into another. + ** + *******************************************************************************/ +public class TableSyncProcess +{ + public static final String FIELD_SOURCE_TABLE_KEY_FIELD = "sourceTableKeyField"; // String + public static final String FIELD_DESTINATION_TABLE_FOREIGN_KEY = "destinationTableForeignKey"; // String + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Builder processMetaDataBuilder(boolean isBasePull) + { + return (Builder) new Builder(StreamedETLWithFrontendProcess.defineProcessMetaData( + isBasePull ? ExtractViaBasepullQueryStep.class : ExtractViaQueryStep.class, + null, + LoadViaInsertOrUpdateStep.class, + Collections.emptyMap())) + .withPreviewStepInputFields(List.of( + new QFieldMetaData(FIELD_SOURCE_TABLE_KEY_FIELD, QFieldType.STRING), + new QFieldMetaData(FIELD_DESTINATION_TABLE_FOREIGN_KEY, QFieldType.STRING) + )) + .withPreviewMessage(StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_INSERT_OR_UPDATE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class Builder extends StreamedETLWithFrontendProcess.Builder + { + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Builder(QProcessMetaData processMetaData) + { + super(processMetaData); + } + + + + /******************************************************************************* + ** Fluent setter for sourceTableKeyField + ** + *******************************************************************************/ + public Builder withSourceTableKeyField(String sourceTableKeyField) + { + setInputFieldDefaultValue(FIELD_SOURCE_TABLE_KEY_FIELD, sourceTableKeyField); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for destinationTableForeignKeyField + ** + *******************************************************************************/ + public Builder withDestinationTableForeignKeyField(String destinationTableForeignKeyField) + { + setInputFieldDefaultValue(FIELD_DESTINATION_TABLE_FOREIGN_KEY, destinationTableForeignKeyField); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for transformStepClass + ** + *******************************************************************************/ + public Builder withSyncTransformStepClass(Class transformStepClass) + { + setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE, new QCodeReference(transformStepClass)); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for sourceTable + ** + *******************************************************************************/ + public Builder withSourceTable(String sourceTable) + { + setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, sourceTable); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for destinationTable + ** + *******************************************************************************/ + public Builder withDestinationTable(String destinationTable) + { + setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, destinationTable); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public Builder withName(String name) + { + processMetaData.setName(name); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public Builder withLabel(String name) + { + processMetaData.setLabel(name); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for tableName + ** + *******************************************************************************/ + public Builder withTableName(String tableName) + { + processMetaData.setTableName(tableName); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for icon + ** + *******************************************************************************/ + public Builder withIcon(QIcon icon) + { + processMetaData.setIcon(icon); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withReviewStepRecordFields(List fieldList) + { + QFrontendStepMetaData reviewStep = processMetaData.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + for(QFieldMetaData fieldMetaData : fieldList) + { + reviewStep.withRecordListField(fieldMetaData); + } + + return (this); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java index 4a244ff7..1999f928 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java @@ -119,7 +119,7 @@ public class ScheduleManager for(QProcessMetaData process : qInstance.getProcesses().values()) { - if(process.getSchedule() != null) + if(process.getSchedule() != null && allowedToStart(process.getName())) { startProcess(process); } @@ -140,33 +140,56 @@ public class ScheduleManager List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions) { - PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableAction); - StandardScheduledExecutor executor = new StandardScheduledExecutor(runner); + if(allowedToStart(tableAction.tableName())) + { + PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableAction); + StandardScheduledExecutor executor = new StandardScheduledExecutor(runner); - QScheduleMetaData schedule = Objects.requireNonNullElseGet(automationProvider.getSchedule(), this::getDefaultSchedule); + QScheduleMetaData schedule = Objects.requireNonNullElseGet(automationProvider.getSchedule(), this::getDefaultSchedule); - executor.setName(runner.getName()); - setScheduleInExecutor(schedule, executor); - executor.start(); + executor.setName(runner.getName()); + setScheduleInExecutor(schedule, executor); + executor.start(); - executors.add(executor); + executors.add(executor); + } } } + /******************************************************************************* + ** + *******************************************************************************/ + private boolean allowedToStart(String name) + { + String propertyName = "qqq.scheduleManager.onlyStartNamesMatching"; + String propertyValue = System.getProperty(propertyName, ""); + if(propertyValue.equals("")) + { + return (true); + } + + return (name.matches(propertyValue)); + } + + + /******************************************************************************* ** *******************************************************************************/ private void startQueueProvider(QQueueProviderMetaData queueProvider) { - switch(queueProvider.getType()) + if(allowedToStart(queueProvider.getName())) { - case SQS: - startSqsProvider((SQSQueueProviderMetaData) queueProvider); - break; - default: - throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); + switch(queueProvider.getType()) + { + case SQS: + startSqsProvider((SQSQueueProviderMetaData) queueProvider); + break; + default: + throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); + } } } @@ -182,7 +205,7 @@ public class ScheduleManager for(QQueueMetaData queue : qInstance.getQueues().values()) { - if(queueProvider.getName().equals(queue.getProviderName())) + if(queueProvider.getName().equals(queue.getProviderName()) && allowedToStart(queue.getName())) { SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); sqsQueuePoller.setQueueProviderMetaData(queueProvider); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java index d1de143d..aa35430a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java @@ -189,6 +189,7 @@ class ExportActionTest { QTableMetaData wideTable = new QTableMetaData() .withName("wide") + .withPrimaryKeyField("field0") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME); for(int i = 0; i < ReportFormat.XLSX.getMaxCols() + 1; i++) { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java index fd79e473..cea36216 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java @@ -129,7 +129,6 @@ public class QPossibleValueTranslatorTest { QInstance qInstance = TestUtils.defineInstance(); QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession()); - QTableMetaData shapeTable = qInstance.getTable(TestUtils.TABLE_NAME_SHAPE); QFieldMetaData shapeField = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId"); QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(shapeField.getPossibleValueSourceName()); @@ -195,6 +194,32 @@ public class QPossibleValueTranslatorTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPossibleValueTableWithBadForeignKeys() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession()); + QFieldMetaData shapeField = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId"); + + TestUtils.insertDefaultShapes(qInstance); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that we don't re-run queries for cached values, even ones that aren't found (e.g., 4 below). // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + MemoryRecordStore.setCollectStatistics(true); + possibleValueTranslator.translatePossibleValue(shapeField, 1); + possibleValueTranslator.translatePossibleValue(shapeField, 2); + assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 queries so far"); + assertNull(possibleValueTranslator.translatePossibleValue(shapeField, 4)); + assertNull(possibleValueTranslator.translatePossibleValue(shapeField, 4)); + assertEquals(3, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 3 queries in total"); + } + + + /******************************************************************************* ** Make sure that if we have 2 different PVS's pointed at the same 1 table, ** that we avoid re-doing queries, and that we actually get different (formatted) values. diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 2d20dbc6..8cbfc987 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -354,10 +354,10 @@ class QInstanceValidatorTest public void test_validateTableWithNoFields() { assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(null), - "At least 1 field"); + "At least 1 field", "Primary key for table person is not a recognized field"); assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(new HashMap<>()), - "At least 1 field"); + "At least 1 field", "Primary key for table person is not a recognized field"); } @@ -547,6 +547,7 @@ class QInstanceValidatorTest { QTableMetaData table = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection(null, "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)); assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "Missing a name"); @@ -562,6 +563,7 @@ class QInstanceValidatorTest { QTableMetaData table = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", null, new QIcon("person"), Tier.T1, List.of("id"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)); assertValidationSuccess((qInstance) -> qInstance.addTable(table)); @@ -577,6 +579,7 @@ class QInstanceValidatorTest { QTableMetaData table = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) .withSection(new QFieldSection("section1", "Section 2", new QIcon("person"), Tier.T2, List.of("name"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)) @@ -594,6 +597,7 @@ class QInstanceValidatorTest { QTableMetaData table = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) .withSection(new QFieldSection("section2", "Section 1", new QIcon("person"), Tier.T2, List.of("name"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)) @@ -611,12 +615,14 @@ class QInstanceValidatorTest { QTableMetaData table1 = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of())) .withField(new QFieldMetaData("id", QFieldType.INTEGER)); assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "section1 does not have any fields", "field id is not listed in any field sections"); QTableMetaData table2 = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, null)) .withField(new QFieldMetaData("id", QFieldType.INTEGER)); assertValidationFailureReasons((qInstance) -> qInstance.addTable(table2), "section1 does not have any fields", "field id is not listed in any field sections"); @@ -632,6 +638,7 @@ class QInstanceValidatorTest { QTableMetaData table = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "od"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)); assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "not a field on this table"); @@ -647,12 +654,14 @@ class QInstanceValidatorTest { QTableMetaData table1 = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "id"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)); assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "more than once"); QTableMetaData table2 = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) .withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T2, List.of("id"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)); @@ -669,6 +678,7 @@ class QInstanceValidatorTest { QTableMetaData table = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)) .withField(new QFieldMetaData("name", QFieldType.STRING)); @@ -685,6 +695,7 @@ class QInstanceValidatorTest { QTableMetaData table = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) .withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T1, List.of("name"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)) @@ -800,6 +811,7 @@ class QInstanceValidatorTest { QTableMetaData table = new QTableMetaData().withName("test") .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) .withField(new QFieldMetaData("id", QFieldType.INTEGER)) .withField(new QFieldMetaData("name", QFieldType.STRING)); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java new file mode 100644 index 00000000..6c06dce1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java @@ -0,0 +1,184 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecordEnum; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for EnumerationCountAction + *******************************************************************************/ +class EnumerationCountActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUnfilteredCount() throws QException + { + QInstance instance = defineQInstance(); + + CountInput countInput = new CountInput(instance); + countInput.setSession(new QSession()); + countInput.setTableName("statesEnum"); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(2, countOutput.getCount()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFilteredCount() throws QException + { + QInstance instance = defineQInstance(); + + CountInput countInput = new CountInput(instance); + countInput.setSession(new QSession()); + countInput.setTableName("statesEnum"); + countInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("population", QCriteriaOperator.GREATER_THAN, List.of(20_000_000)))); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(1, countOutput.getCount()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QInstance defineQInstance() + { + QInstance instance = TestUtils.defineInstance(); + instance.addBackend(new QBackendMetaData() + .withName("enum") + .withBackendType("enum") + ); + + instance.addTable(new QTableMetaData() + .withName("statesEnum") + .withBackendName("enum") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + .withField(new QFieldMetaData("postalCode", QFieldType.STRING)) + .withField(new QFieldMetaData("population", QFieldType.INTEGER)) + .withBackendDetails(new EnumerationTableBackendDetails().withEnumClass(States.class)) + ); + return instance; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static enum States implements QRecordEnum + { + MO(1, "Missouri", "MO", 15_000_000), + IL(2, "Illinois", "IL", 25_000_000); + + + private final Integer id; + private final String name; + private final String postalCode; + private final Integer population; + + + + /******************************************************************************* + ** + *******************************************************************************/ + States(int id, String name, String postalCode, int population) + { + this.id = id; + this.name = name; + this.postalCode = postalCode; + this.population = population; + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Getter for postalCode + ** + *******************************************************************************/ + public String getPostalCode() + { + return postalCode; + } + + + + /******************************************************************************* + ** Getter for population + ** + *******************************************************************************/ + public Integer getPopulation() + { + return population; + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java new file mode 100644 index 00000000..caaa28e9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java @@ -0,0 +1,147 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.tablesync; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for TableSyncProcess + *******************************************************************************/ +class TableSyncProcessTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + QTableMetaData personTable = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + String TABLE_NAME_PEOPLE_SYNC = "peopleSync"; + qInstance.addTable(new QTableMetaData() + .withName(TABLE_NAME_PEOPLE_SYNC) + .withPrimaryKeyField("id") + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withFields(personTable.getFields()) + .withField(new QFieldMetaData("sourcePersonId", QFieldType.INTEGER))); + + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Darin"), + new QRecord().withValue("id", 2).withValue("firstName", "Tim"), + new QRecord().withValue("id", 3).withValue("firstName", "Tyler"), + new QRecord().withValue("id", 4).withValue("firstName", "James"), + new QRecord().withValue("id", 5).withValue("firstName", "Homer") + )); + + TestUtils.insertRecords(qInstance, qInstance.getTable(TABLE_NAME_PEOPLE_SYNC), List.of( + new QRecord().withValue("sourcePersonId", 3).withValue("firstName", "Garret"), + new QRecord().withValue("sourcePersonId", 5).withValue("firstName", "Homer") + )); + + String PROCESS_NAME = "testSyncProcess"; + qInstance.addProcess(TableSyncProcess.processMetaDataBuilder(false) + .withName(PROCESS_NAME) + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withDestinationTable(TABLE_NAME_PEOPLE_SYNC) + .withSourceTableKeyField("id") + .withDestinationTableForeignKeyField("sourcePersonId") + .withSyncTransformStepClass(PersonTransformClass.class) + .getProcessMetaData()); + + RunProcessInput runProcessInput = new RunProcessInput(qInstance); + runProcessInput.setSession(new QSession()); + runProcessInput.setProcessName(PROCESS_NAME); + runProcessInput.addValue("recordIds", "1,2,3,4,5"); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + + RunProcessAction runProcessAction = new RunProcessAction(); + RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput); + + @SuppressWarnings("unchecked") + ArrayList processResults = (ArrayList) runProcessOutput.getValues().get("processResults"); + + assertThat(processResults.get(0)) + .hasFieldOrPropertyWithValue("message", "were inserted") + .hasFieldOrPropertyWithValue("count", 3); + + assertThat(processResults.get(1)) + .hasFieldOrPropertyWithValue("message", "were updated") + .hasFieldOrPropertyWithValue("count", 2); + + List syncedRecords = TestUtils.queryTable(qInstance, TABLE_NAME_PEOPLE_SYNC); + assertEquals(5, syncedRecords.size()); + + ///////////////////////////////////////////////////////////////// + // make sure the record referencing 3 has had its name updated // + // and the one referencing 5 stayed the same // + ///////////////////////////////////////////////////////////////// + Map syncPersonsBySourceId = GeneralProcessUtils.loadTableToMap(runProcessInput, TABLE_NAME_PEOPLE_SYNC, "sourcePersonId"); + assertEquals("Tyler", syncPersonsBySourceId.get(3).getValueString("firstName")); + assertEquals("Homer", syncPersonsBySourceId.get(5).getValueString("firstName")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class PersonTransformClass extends AbstractTableSyncTransformStep + { + + @Override + public QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException + { + destinationRecord.setValue("sourcePersonId", sourceRecord.getValue("id")); + destinationRecord.setValue("firstName", sourceRecord.getValue("firstName")); + destinationRecord.setValue("lastName", sourceRecord.getValue("lastName")); + return (destinationRecord); + } + + } + +} \ No newline at end of file diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIGetAction.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIGetAction.java index ccae3a18..f91cf076 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIGetAction.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIGetAction.java @@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; @@ -60,16 +61,28 @@ public class APIGetAction extends AbstractAPIAction implements GetInterface String url = apiActionUtil.buildTableUrl(table); HttpGet request = new HttpGet(url + urlSuffix); + LOG.debug("GET " + url + urlSuffix); + apiActionUtil.setupAuthorizationInRequest(request); apiActionUtil.setupContentTypeInRequest(request); apiActionUtil.setupAdditionalHeaders(request); try(CloseableHttpResponse response = httpClient.execute(request)) { - QRecord record = apiActionUtil.processSingleRecordGetResponse(table, response); - GetOutput rs = new GetOutput(); - rs.setRecord(record); + if(response.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_FOUND) + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // leave get response null - downstream will convert into not-found exception if/as needed // + ///////////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("HTTP GET for " + table.getName() + " " + getInput.getPrimaryKey() + " failed with status 404."); + } + else + { + QRecord record = apiActionUtil.processSingleRecordGetResponse(table, response); + rs.setRecord(record); + } + return rs; } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java index 5cf43010..76add5c0 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java @@ -76,4 +76,15 @@ public class AbstractFilesystemBackendMetaData extends QBackendMetaData return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean requiresPrimaryKeyOnTables() + { + return (false); + } + }