Add TableSyncProcess

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -414,6 +414,9 @@ public class QPossibleValueTranslator
 
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
+            ///////////////////////////////////////////////////////////////////////////////////
+            // for all records that were found, put a formatted value into cache foreach PVS //
+            ///////////////////////////////////////////////////////////////////////////////////
             for(QRecord record : queryOutput.getRecords())
             {
                Serializable pkeyValue = record.getValue(primaryKeyField);
@@ -423,6 +426,20 @@ public class QPossibleValueTranslator
                   possibleValueCache.get(possibleValueSource.getName()).put(pkeyValue, formatPossibleValue(possibleValueSource, possibleValue));
                }
             }
+
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // for all pkeys that were NOT found, put a null value into cache foreach PVS (to avoid re-looking up) //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////
+            for(Serializable pkey : page)
+            {
+               for(QPossibleValueSource possibleValueSource : possibleValueSources)
+               {
+                  if(!possibleValueCache.get(possibleValueSource.getName()).containsKey(pkey))
+                  {
+                     possibleValueCache.get(possibleValueSource.getName()).put(pkey, null);
+                  }
+               }
+            }
          }
       }
       catch(Exception e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -36,11 +36,13 @@ import java.util.stream.Stream;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
@@ -55,6 +57,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
@@ -222,24 +225,34 @@ public class QInstanceValidator
     *******************************************************************************/
    private void validateTables(QInstance qInstance)
    {
-      if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()),
+      if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()), "At least 1 table must be defined."))
-         "At least 1 table must be defined."))
       {
          qInstance.getTables().forEach((tableName, table) ->
          {
             assertCondition(Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + ".");
-
             validateAppChildHasValidParentAppName(qInstance, table);
 
             ////////////////////////////////////////
             // validate the backend for the table //
             ////////////////////////////////////////
-            if(assertCondition(StringUtils.hasContent(table.getBackendName()),
+            if(assertCondition(StringUtils.hasContent(table.getBackendName()), "Missing backend name for table " + tableName + "."))
-               "Missing backend name for table " + tableName + "."))
             {
                if(CollectionUtils.nullSafeHasContents(qInstance.getBackends()))
                {
-                  assertCondition(qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + ".");
+                  QBackendMetaData backendForTable = qInstance.getBackendForTable(tableName);
+                  if(assertCondition(backendForTable != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + "."))
+                  {
+                     ////////////////////////////////////////////////////////////
+                     // if the backend requires primary keys, then validate it //
+                     ////////////////////////////////////////////////////////////
+                     if(backendForTable.requiresPrimaryKeyOnTables())
+                     {
+                        if(assertCondition(StringUtils.hasContent(table.getPrimaryKeyField()), "Missing primary key for table: " + tableName))
+                        {
+                           assertNoException(() -> table.getField(table.getPrimaryKeyField()), "Primary key for table " + tableName + " is not a recognized field on this table.");
+                        }
+                     }
+                  }
                }
             }
 
@@ -329,12 +342,49 @@ public class QInstanceValidator
             {
                validateTableUniqueKeys(table);
             }
+
+            /////////////////////////////////////////////
+            // validate the table's associated scripts //
+            /////////////////////////////////////////////
+            if(table.getAssociatedScripts() != null)
+            {
+               validateAssociatedScripts(table);
+            }
          });
       }
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void validateAssociatedScripts(QTableMetaData table)
+   {
+      Set<String> usedFieldNames = new HashSet<>();
+      for(AssociatedScript associatedScript : table.getAssociatedScripts())
+      {
+         if(assertCondition(StringUtils.hasContent(associatedScript.getFieldName()), "Table " + table.getName() + " has an associatedScript without a fieldName"))
+         {
+            assertCondition(!usedFieldNames.contains(associatedScript.getFieldName()), "Table " + table.getName() + " has more than one associatedScript specifying field: " + associatedScript.getFieldName());
+            usedFieldNames.add(associatedScript.getFieldName());
+            assertNoException(() -> table.getField(associatedScript.getFieldName()), "Table " + table.getName() + " has an associatedScript specifying an unrecognized field: " + associatedScript.getFieldName());
+         }
+
+         assertCondition(associatedScript.getScriptTypeId() != null, "Table " + table.getName() + " associatedScript on field " + associatedScript.getFieldName() + " is missing a scriptTypeId");
+         if(associatedScript.getScriptTester() != null)
+         {
+            String prefix = "Table " + table.getName() + " associatedScript on field " + associatedScript.getFieldName();
+            if(preAssertionsForCodeReference(associatedScript.getScriptTester(), prefix))
+            {
+               validateSimpleCodeReference(prefix, associatedScript.getScriptTester(), TestScriptActionInterface.class);
+            }
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java

@@ -106,6 +106,7 @@ public class RunBackendStepInput extends AbstractActionInput
       target.setTableName(getTableName());
       target.setProcessName(getProcessName());
       target.setAsyncJobCallback(getAsyncJobCallback());
+      target.setFrontendStepBehavior(getFrontendStepBehavior());
       target.setValues(getValues());
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java

@@ -35,6 +35,8 @@ import java.util.Locale;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -42,6 +44,8 @@ import com.kingsrook.qqq.backend.core.utils.ListingHash;
  *******************************************************************************/
 public interface QRecordEnum
 {
+   Logger LOG = LogManager.getLogger(QRecordEnum.class);
+
    ListingHash<Class<? extends QRecordEnum>, QRecordEntityField> fieldMapping = new ListingHash<>();
 
 
@@ -140,9 +144,9 @@ public interface QRecordEnum
          }
          else
          {
-            if(!method.getName().equals("getClass"))
+            if(!method.getName().equals("getClass") && !method.getName().equals("getDeclaringClass") && !method.getName().equals("getPossibleValueId"))
             {
-               System.err.println("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported.");
+               LOG.debug("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported.");
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -60,6 +60,16 @@ public class QBackendMetaData
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public boolean requiresPrimaryKeyOnTables()
+   {
+      return (true);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration;
 
 
+import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -68,4 +69,15 @@ public class EnumerationBackendModule implements QBackendModuleInterface
       return new EnumerationQueryAction();
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public CountInterface getCountInterface()
+   {
+      return new EnumerationCountAction();
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountAction.java

@@ -0,0 +1,57 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration;
+
+
+import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class EnumerationCountAction implements CountInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public CountOutput execute(CountInput countInput) throws QException
+   {
+      QueryInput queryInput = new QueryInput(countInput.getInstance());
+      queryInput.setSession(countInput.getSession());
+      queryInput.setTableName(countInput.getTableName());
+      queryInput.setFilter(countInput.getFilter());
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+     
+      CountOutput countOutput = new CountOutput();
+      countOutput.setCount(queryOutput.getRecords().size());
+      return (countOutput);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java

@@ -273,6 +273,14 @@ public class BackendQueryFilterUtils
     *******************************************************************************/
    private static boolean testIn(QFilterCriteria criterion, Serializable value)
    {
+      if(CollectionUtils.nullSafeHasContents(criterion.getValues()))
+      {
+         if(criterion.getValues().get(0) instanceof String && value instanceof Number)
+         {
+            value = String.valueOf(value);
+         }
+      }
+
       if(!criterion.getValues().contains(value))
       {
          return (false);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java

@@ -373,5 +373,21 @@ public class StreamedETLWithFrontendProcess
 
          return (this);
       }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withPreviewStepInputFields(List<QFieldMetaData> fieldList)
+      {
+         QBackendStepMetaData previewStep = processMetaData.getBackendStep(STEP_NAME_PREVIEW);
+         for(QFieldMetaData field : fieldList)
+         {
+            previewStep.getInputMetaData().withField(field);
+         }
+
+         return (this);
+      }
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/StandardProcessSummaryLineProducer.java

@@ -0,0 +1,80 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.general;
+
+
+import java.util.ArrayList;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import static com.kingsrook.qqq.backend.core.model.actions.processes.Status.OK;
+
+
+/*******************************************************************************
+ ** Helper for working with process summary lines
+ *******************************************************************************/
+public class StandardProcessSummaryLineProducer
+{
+
+   /*******************************************************************************
+    ** Make a line that'll say " {will be/was/were} inserted"
+    *******************************************************************************/
+   public static ProcessSummaryLine getOkToInsertLine()
+   {
+      return new ProcessSummaryLine(OK)
+         .withMessageSuffix(" inserted")
+         .withSingularFutureMessage("will be")
+         .withPluralFutureMessage("will be")
+         .withSingularPastMessage("was")
+         .withPluralPastMessage("were");
+   }
+
+
+
+   /*******************************************************************************
+    ** Make a line that'll say " {will be/was/were} updated"
+    *******************************************************************************/
+   public static ProcessSummaryLine getOkToUpdateLine()
+   {
+      return new ProcessSummaryLine(OK)
+         .withMessageSuffix(" updated")
+         .withSingularFutureMessage("will be")
+         .withPluralFutureMessage("will be")
+         .withSingularPastMessage("was")
+         .withPluralPastMessage("were");
+   }
+
+
+
+   /*******************************************************************************
+    ** one-liner for implementing getProcessSummary - just pass your lines in as varargs as in:
+    ** return (StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate));
+    *******************************************************************************/
+   public static ArrayList<ProcessSummaryLineInterface> toArrayList(ProcessSummaryLine... lines)
+   {
+      ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
+      for(ProcessSummaryLine line : lines)
+      {
+         line.addSelfToListIfAnyCount(rs);
+      }
+      return (rs);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java

@@ -0,0 +1,230 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer;
+import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** This class is for transforming records from a Source table to a Destination table.
+ **
+ ** The Source table has a (unique/primary) key field:  sourceTableKeyField,
+ ** Which is matched against the Destination table's foreign-key:  destinationTableForeignKeyField
+ *******************************************************************************/
+public abstract class AbstractTableSyncTransformStep extends AbstractTransformStep
+{
+   private ProcessSummaryLine okToInsert           = StandardProcessSummaryLineProducer.getOkToInsertLine();
+   private ProcessSummaryLine okToUpdate           = StandardProcessSummaryLineProducer.getOkToUpdateLine();
+   private ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR)
+      .withMessageSuffix("missing a value for the key field.")
+      .withSingularFutureMessage("will not be synced, because it is ")
+      .withPluralFutureMessage("will not be synced, because they are ")
+      .withSingularPastMessage("was not synced, because it is ")
+      .withPluralPastMessage("were not synced, because they are ");
+
+   private RunBackendStepInput runBackendStepInput = null;
+
+   private QPossibleValueTranslator possibleValueTranslator;
+
+   private Map<String, Map<Serializable, QRecord>> tableMaps = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected QRecord getRecord(String tableName, String fieldName, Serializable value) throws QException
+   {
+      if(!tableMaps.containsKey(tableName))
+      {
+         Map<Serializable, QRecord> recordMap = GeneralProcessUtils.loadTableToMap(runBackendStepInput, tableName, fieldName);
+         tableMaps.put(tableName, recordMap);
+      }
+
+      return (tableMaps.get(tableName).get(value));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected Serializable getRecordField(String tableName, String fieldName, Serializable value, String outputField) throws QException
+   {
+      QRecord record = getRecord(tableName, fieldName, value);
+      if(record == null)
+      {
+         return (null);
+      }
+
+      return (record.getValue(outputField));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
+   {
+      return (StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords()))
+      {
+         return;
+      }
+
+      this.runBackendStepInput = runBackendStepInput;
+      String sourceTableKeyField             = runBackendStepInput.getValueString(TableSyncProcess.FIELD_SOURCE_TABLE_KEY_FIELD);
+      String destinationTableForeignKeyField = runBackendStepInput.getValueString(TableSyncProcess.FIELD_DESTINATION_TABLE_FOREIGN_KEY);
+      String destinationTableName            = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE);
+
+      //////////////////////////////////////
+      // extract keys from source records //
+      //////////////////////////////////////
+      List<Serializable> sourceKeyList = runBackendStepInput.getRecords().stream()
+         .map(r -> r.getValueString(sourceTableKeyField))
+         .filter(Objects::nonNull)
+         .filter(v -> !"".equals(v))
+         .collect(Collectors.toList());
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // query to see if we already have those records in the destination (to determine insert/update) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      Map<Serializable, QRecord> existingRecordsByForeignKey = Collections.emptyMap();
+      if(!sourceKeyList.isEmpty())
+      {
+         QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance());
+         queryInput.setSession(runBackendStepInput.getSession());
+         queryInput.setTableName(destinationTableName);
+         queryInput.setFilter(new QQueryFilter()
+            .withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList))
+         );
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         existingRecordsByForeignKey = CollectionUtils.recordsToMap(queryOutput.getRecords(), destinationTableForeignKeyField);
+      }
+
+      /////////////////////////////////////////////////////////////////
+      // foreach source record, build the record we'll insert/update //
+      /////////////////////////////////////////////////////////////////
+      for(QRecord sourceRecord : runBackendStepInput.getRecords())
+      {
+         Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField);
+         QRecord      existingRecord = existingRecordsByForeignKey.get(sourceKeyValue);
+
+         if(sourceKeyValue == null || "".equals(sourceKeyValue))
+         {
+            errorMissingKeyField.incrementCount();
+
+            try
+            {
+               errorMissingKeyField.setMessageSuffix("missing a value for the field " + runBackendStepInput.getTable().getField(sourceTableKeyField).getLabel());
+            }
+            catch(Exception e)
+            {
+               /////////////////////////////////////////
+               // just leave the default error suffix //
+               /////////////////////////////////////////
+            }
+
+            continue;
+         }
+
+         QRecord recordToStore;
+         if(existingRecord != null)
+         {
+            recordToStore = existingRecord;
+            okToUpdate.incrementCount();
+         }
+         else
+         {
+            recordToStore = new QRecord();
+            okToInsert.incrementCount();
+         }
+
+         recordToStore = populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord);
+         runBackendStepOutput.addRecord(recordToStore);
+      }
+
+      ////////////////////////////////////////////////
+      // populate possible-values for review screen //
+      ////////////////////////////////////////////////
+      if(RunProcessInput.FrontendStepBehavior.BREAK.equals(runBackendStepInput.getFrontendStepBehavior()))
+      {
+         if(CollectionUtils.nullSafeHasContents(runBackendStepOutput.getRecords()))
+         {
+            if(possibleValueTranslator == null)
+            {
+               possibleValueTranslator = new QPossibleValueTranslator(runBackendStepInput.getInstance(), runBackendStepInput.getSession());
+            }
+
+            possibleValueTranslator.translatePossibleValuesInRecords(runBackendStepInput.getInstance().getTable(destinationTableName), runBackendStepOutput.getRecords());
+         }
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java

@@ -0,0 +1,208 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
+
+
+import java.util.Collections;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertOrUpdateStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+
+
+/*******************************************************************************
+ ** Definition for Standard process to sync data from one table into another.
+ **
+ *******************************************************************************/
+public class TableSyncProcess
+{
+   public static final String FIELD_SOURCE_TABLE_KEY_FIELD        = "sourceTableKeyField"; // String
+   public static final String FIELD_DESTINATION_TABLE_FOREIGN_KEY = "destinationTableForeignKey"; // String
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Builder processMetaDataBuilder(boolean isBasePull)
+   {
+      return (Builder) new Builder(StreamedETLWithFrontendProcess.defineProcessMetaData(
+         isBasePull ? ExtractViaBasepullQueryStep.class : ExtractViaQueryStep.class,
+         null,
+         LoadViaInsertOrUpdateStep.class,
+         Collections.emptyMap()))
+         .withPreviewStepInputFields(List.of(
+            new QFieldMetaData(FIELD_SOURCE_TABLE_KEY_FIELD, QFieldType.STRING),
+            new QFieldMetaData(FIELD_DESTINATION_TABLE_FOREIGN_KEY, QFieldType.STRING)
+         ))
+         .withPreviewMessage(StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_INSERT_OR_UPDATE);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class Builder extends StreamedETLWithFrontendProcess.Builder
+   {
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public Builder(QProcessMetaData processMetaData)
+      {
+         super(processMetaData);
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for sourceTableKeyField
+       **
+       *******************************************************************************/
+      public Builder withSourceTableKeyField(String sourceTableKeyField)
+      {
+         setInputFieldDefaultValue(FIELD_SOURCE_TABLE_KEY_FIELD, sourceTableKeyField);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for destinationTableForeignKeyField
+       **
+       *******************************************************************************/
+      public Builder withDestinationTableForeignKeyField(String destinationTableForeignKeyField)
+      {
+         setInputFieldDefaultValue(FIELD_DESTINATION_TABLE_FOREIGN_KEY, destinationTableForeignKeyField);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for transformStepClass
+       **
+       *******************************************************************************/
+      public Builder withSyncTransformStepClass(Class<? extends AbstractTableSyncTransformStep> transformStepClass)
+      {
+         setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE, new QCodeReference(transformStepClass));
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for sourceTable
+       **
+       *******************************************************************************/
+      public Builder withSourceTable(String sourceTable)
+      {
+         setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, sourceTable);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for destinationTable
+       **
+       *******************************************************************************/
+      public Builder withDestinationTable(String destinationTable)
+      {
+         setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, destinationTable);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for name
+       **
+       *******************************************************************************/
+      public Builder withName(String name)
+      {
+         processMetaData.setName(name);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for label
+       **
+       *******************************************************************************/
+      public Builder withLabel(String name)
+      {
+         processMetaData.setLabel(name);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for tableName
+       **
+       *******************************************************************************/
+      public Builder withTableName(String tableName)
+      {
+         processMetaData.setTableName(tableName);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for icon
+       **
+       *******************************************************************************/
+      public Builder withIcon(QIcon icon)
+      {
+         processMetaData.setIcon(icon);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withReviewStepRecordFields(List<QFieldMetaData> fieldList)
+      {
+         QFrontendStepMetaData reviewStep = processMetaData.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW);
+         for(QFieldMetaData fieldMetaData : fieldList)
+         {
+            reviewStep.withRecordListField(fieldMetaData);
+         }
+
+         return (this);
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java

@@ -119,7 +119,7 @@ public class ScheduleManager
 
       for(QProcessMetaData process : qInstance.getProcesses().values())
       {
-         if(process.getSchedule() != null)
+         if(process.getSchedule() != null && allowedToStart(process.getName()))
          {
             startProcess(process);
          }
@@ -140,33 +140,56 @@ public class ScheduleManager
       List<PollingAutomationPerTableRunner.TableActions> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName());
       for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions)
       {
-         PollingAutomationPerTableRunner runner   = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableAction);
+         if(allowedToStart(tableAction.tableName()))
-         StandardScheduledExecutor       executor = new StandardScheduledExecutor(runner);
+         {
+            PollingAutomationPerTableRunner runner   = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableAction);
+            StandardScheduledExecutor       executor = new StandardScheduledExecutor(runner);
 
-         QScheduleMetaData schedule = Objects.requireNonNullElseGet(automationProvider.getSchedule(), this::getDefaultSchedule);
+            QScheduleMetaData schedule = Objects.requireNonNullElseGet(automationProvider.getSchedule(), this::getDefaultSchedule);
 
-         executor.setName(runner.getName());
+            executor.setName(runner.getName());
-         setScheduleInExecutor(schedule, executor);
+            setScheduleInExecutor(schedule, executor);
-         executor.start();
+            executor.start();
 
-         executors.add(executor);
+            executors.add(executor);
+         }
       }
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean allowedToStart(String name)
+   {
+      String propertyName  = "qqq.scheduleManager.onlyStartNamesMatching";
+      String propertyValue = System.getProperty(propertyName, "");
+      if(propertyValue.equals(""))
+      {
+         return (true);
+      }
+
+      return (name.matches(propertyValue));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    private void startQueueProvider(QQueueProviderMetaData queueProvider)
    {
-      switch(queueProvider.getType())
+      if(allowedToStart(queueProvider.getName()))
       {
-         case SQS:
+         switch(queueProvider.getType())
-            startSqsProvider((SQSQueueProviderMetaData) queueProvider);
+         {
-            break;
+            case SQS:
-         default:
+               startSqsProvider((SQSQueueProviderMetaData) queueProvider);
-            throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType());
+               break;
+            default:
+               throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType());
+         }
       }
    }
 
@@ -182,7 +205,7 @@ public class ScheduleManager
 
       for(QQueueMetaData queue : qInstance.getQueues().values())
       {
-         if(queueProvider.getName().equals(queue.getProviderName()))
+         if(queueProvider.getName().equals(queue.getProviderName()) && allowedToStart(queue.getName()))
          {
             SQSQueuePoller sqsQueuePoller = new SQSQueuePoller();
             sqsQueuePoller.setQueueProviderMetaData(queueProvider);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java

@@ -189,6 +189,7 @@ class ExportActionTest
    {
       QTableMetaData wideTable = new QTableMetaData()
          .withName("wide")
+         .withPrimaryKeyField("field0")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME);
       for(int i = 0; i < ReportFormat.XLSX.getMaxCols() + 1; i++)
       {

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java

@@ -129,7 +129,6 @@ public class QPossibleValueTranslatorTest
    {
       QInstance                qInstance               = TestUtils.defineInstance();
       QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
-      QTableMetaData           shapeTable              = qInstance.getTable(TestUtils.TABLE_NAME_SHAPE);
       QFieldMetaData           shapeField              = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId");
       QPossibleValueSource     possibleValueSource     = qInstance.getPossibleValueSource(shapeField.getPossibleValueSourceName());
 
@@ -195,6 +194,32 @@ public class QPossibleValueTranslatorTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPossibleValueTableWithBadForeignKeys() throws QException
+   {
+      QInstance                qInstance               = TestUtils.defineInstance();
+      QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
+      QFieldMetaData           shapeField              = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId");
+
+      TestUtils.insertDefaultShapes(qInstance);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // assert that we don't re-run queries for cached values, even ones that aren't found (e.g., 4 below). //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      MemoryRecordStore.setCollectStatistics(true);
+      possibleValueTranslator.translatePossibleValue(shapeField, 1);
+      possibleValueTranslator.translatePossibleValue(shapeField, 2);
+      assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 queries so far");
+      assertNull(possibleValueTranslator.translatePossibleValue(shapeField, 4));
+      assertNull(possibleValueTranslator.translatePossibleValue(shapeField, 4));
+      assertEquals(3, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 3 queries in total");
+   }
+
+
+
    /*******************************************************************************
     ** Make sure that if we have 2 different PVS's pointed at the same 1 table,
     ** that we avoid re-doing queries, and that we actually get different (formatted) values.

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -354,10 +354,10 @@ class QInstanceValidatorTest
    public void test_validateTableWithNoFields()
    {
       assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(null),
-         "At least 1 field");
+         "At least 1 field", "Primary key for table person is not a recognized field");
 
       assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(new HashMap<>()),
-         "At least 1 field");
+         "At least 1 field", "Primary key for table person is not a recognized field");
    }
 
 
@@ -547,6 +547,7 @@ class QInstanceValidatorTest
    {
       QTableMetaData table = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection(null, "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER));
       assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "Missing a name");
@@ -562,6 +563,7 @@ class QInstanceValidatorTest
    {
       QTableMetaData table = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", null, new QIcon("person"), Tier.T1, List.of("id")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER));
       assertValidationSuccess((qInstance) -> qInstance.addTable(table));
@@ -577,6 +579,7 @@ class QInstanceValidatorTest
    {
       QTableMetaData table = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
          .withSection(new QFieldSection("section1", "Section 2", new QIcon("person"), Tier.T2, List.of("name")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
@@ -594,6 +597,7 @@ class QInstanceValidatorTest
    {
       QTableMetaData table = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
          .withSection(new QFieldSection("section2", "Section 1", new QIcon("person"), Tier.T2, List.of("name")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
@@ -611,12 +615,14 @@ class QInstanceValidatorTest
    {
       QTableMetaData table1 = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of()))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER));
       assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "section1 does not have any fields", "field id is not listed in any field sections");
 
       QTableMetaData table2 = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, null))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER));
       assertValidationFailureReasons((qInstance) -> qInstance.addTable(table2), "section1 does not have any fields", "field id is not listed in any field sections");
@@ -632,6 +638,7 @@ class QInstanceValidatorTest
    {
       QTableMetaData table = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "od")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER));
       assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "not a field on this table");
@@ -647,12 +654,14 @@ class QInstanceValidatorTest
    {
       QTableMetaData table1 = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "id")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER));
       assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "more than once");
 
       QTableMetaData table2 = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
          .withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T2, List.of("id")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER));
@@ -669,6 +678,7 @@ class QInstanceValidatorTest
    {
       QTableMetaData table = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
          .withField(new QFieldMetaData("name", QFieldType.STRING));
@@ -685,6 +695,7 @@ class QInstanceValidatorTest
    {
       QTableMetaData table = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
          .withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T1, List.of("name")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
@@ -800,6 +811,7 @@ class QInstanceValidatorTest
    {
       QTableMetaData table = new QTableMetaData().withName("test")
          .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
          .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
          .withField(new QFieldMetaData("name", QFieldType.STRING));

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java

@@ -0,0 +1,184 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEnum;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for EnumerationCountAction
+ *******************************************************************************/
+class EnumerationCountActionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testUnfilteredCount() throws QException
+   {
+      QInstance instance = defineQInstance();
+
+      CountInput countInput = new CountInput(instance);
+      countInput.setSession(new QSession());
+      countInput.setTableName("statesEnum");
+      CountOutput countOutput = new CountAction().execute(countInput);
+      assertEquals(2, countOutput.getCount());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFilteredCount() throws QException
+   {
+      QInstance instance = defineQInstance();
+
+      CountInput countInput = new CountInput(instance);
+      countInput.setSession(new QSession());
+      countInput.setTableName("statesEnum");
+      countInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("population", QCriteriaOperator.GREATER_THAN, List.of(20_000_000))));
+      CountOutput countOutput = new CountAction().execute(countInput);
+      assertEquals(1, countOutput.getCount());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QInstance defineQInstance()
+   {
+      QInstance instance = TestUtils.defineInstance();
+      instance.addBackend(new QBackendMetaData()
+         .withName("enum")
+         .withBackendType("enum")
+      );
+
+      instance.addTable(new QTableMetaData()
+         .withName("statesEnum")
+         .withBackendName("enum")
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("name", QFieldType.STRING))
+         .withField(new QFieldMetaData("postalCode", QFieldType.STRING))
+         .withField(new QFieldMetaData("population", QFieldType.INTEGER))
+         .withBackendDetails(new EnumerationTableBackendDetails().withEnumClass(States.class))
+      );
+      return instance;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static enum States implements QRecordEnum
+   {
+      MO(1, "Missouri", "MO", 15_000_000),
+      IL(2, "Illinois", "IL", 25_000_000);
+
+
+      private final Integer id;
+      private final String  name;
+      private final String  postalCode;
+      private final Integer population;
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      States(int id, String name, String postalCode, int population)
+      {
+         this.id = id;
+         this.name = name;
+         this.postalCode = postalCode;
+         this.population = population;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for id
+       **
+       *******************************************************************************/
+      public Integer getId()
+      {
+         return id;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for name
+       **
+       *******************************************************************************/
+      public String getName()
+      {
+         return name;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for postalCode
+       **
+       *******************************************************************************/
+      public String getPostalCode()
+      {
+         return postalCode;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for population
+       **
+       *******************************************************************************/
+      public Integer getPopulation()
+      {
+         return population;
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java

@@ -0,0 +1,147 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for TableSyncProcess
+ *******************************************************************************/
+class TableSyncProcessTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      QTableMetaData personTable = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      String TABLE_NAME_PEOPLE_SYNC = "peopleSync";
+      qInstance.addTable(new QTableMetaData()
+         .withName(TABLE_NAME_PEOPLE_SYNC)
+         .withPrimaryKeyField("id")
+         .withBackendName(TestUtils.MEMORY_BACKEND_NAME)
+         .withFields(personTable.getFields())
+         .withField(new QFieldMetaData("sourcePersonId", QFieldType.INTEGER)));
+
+      TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of(
+         new QRecord().withValue("id", 1).withValue("firstName", "Darin"),
+         new QRecord().withValue("id", 2).withValue("firstName", "Tim"),
+         new QRecord().withValue("id", 3).withValue("firstName", "Tyler"),
+         new QRecord().withValue("id", 4).withValue("firstName", "James"),
+         new QRecord().withValue("id", 5).withValue("firstName", "Homer")
+      ));
+
+      TestUtils.insertRecords(qInstance, qInstance.getTable(TABLE_NAME_PEOPLE_SYNC), List.of(
+         new QRecord().withValue("sourcePersonId", 3).withValue("firstName", "Garret"),
+         new QRecord().withValue("sourcePersonId", 5).withValue("firstName", "Homer")
+      ));
+
+      String PROCESS_NAME = "testSyncProcess";
+      qInstance.addProcess(TableSyncProcess.processMetaDataBuilder(false)
+         .withName(PROCESS_NAME)
+         .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .withDestinationTable(TABLE_NAME_PEOPLE_SYNC)
+         .withSourceTableKeyField("id")
+         .withDestinationTableForeignKeyField("sourcePersonId")
+         .withSyncTransformStepClass(PersonTransformClass.class)
+         .getProcessMetaData());
+
+      RunProcessInput runProcessInput = new RunProcessInput(qInstance);
+      runProcessInput.setSession(new QSession());
+      runProcessInput.setProcessName(PROCESS_NAME);
+      runProcessInput.addValue("recordIds", "1,2,3,4,5");
+      runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+
+      RunProcessAction runProcessAction = new RunProcessAction();
+      RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput);
+
+      @SuppressWarnings("unchecked")
+      ArrayList<ProcessSummaryLineInterface> processResults = (ArrayList<ProcessSummaryLineInterface>) runProcessOutput.getValues().get("processResults");
+
+      assertThat(processResults.get(0))
+         .hasFieldOrPropertyWithValue("message", "were inserted")
+         .hasFieldOrPropertyWithValue("count", 3);
+
+      assertThat(processResults.get(1))
+         .hasFieldOrPropertyWithValue("message", "were updated")
+         .hasFieldOrPropertyWithValue("count", 2);
+
+      List<QRecord> syncedRecords = TestUtils.queryTable(qInstance, TABLE_NAME_PEOPLE_SYNC);
+      assertEquals(5, syncedRecords.size());
+
+      /////////////////////////////////////////////////////////////////
+      // make sure the record referencing 3 has had its name updated //
+      // and the one referencing 5 stayed the same                   //
+      /////////////////////////////////////////////////////////////////
+      Map<Serializable, QRecord> syncPersonsBySourceId = GeneralProcessUtils.loadTableToMap(runProcessInput, TABLE_NAME_PEOPLE_SYNC, "sourcePersonId");
+      assertEquals("Tyler", syncPersonsBySourceId.get(3).getValueString("firstName"));
+      assertEquals("Homer", syncPersonsBySourceId.get(5).getValueString("firstName"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class PersonTransformClass extends AbstractTableSyncTransformStep
+   {
+
+      @Override
+      public QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException
+      {
+         destinationRecord.setValue("sourcePersonId", sourceRecord.getValue("id"));
+         destinationRecord.setValue("firstName", sourceRecord.getValue("firstName"));
+         destinationRecord.setValue("lastName", sourceRecord.getValue("lastName"));
+         return (destinationRecord);
+      }
+
+   }
+
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIGetAction.java

@@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -60,16 +61,28 @@ public class APIGetAction extends AbstractAPIAction implements GetInterface
          String  url     = apiActionUtil.buildTableUrl(table);
          HttpGet request = new HttpGet(url + urlSuffix);
 
+         LOG.debug("GET " + url + urlSuffix);
+
          apiActionUtil.setupAuthorizationInRequest(request);
          apiActionUtil.setupContentTypeInRequest(request);
          apiActionUtil.setupAdditionalHeaders(request);
 
          try(CloseableHttpResponse response = httpClient.execute(request))
          {
-            QRecord record = apiActionUtil.processSingleRecordGetResponse(table, response);
-
             GetOutput rs = new GetOutput();
-            rs.setRecord(record);
+            if(response.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_FOUND)
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               // leave get response null - downstream will convert into not-found exception if/as needed //
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               LOG.debug("HTTP GET for " + table.getName() + " " + getInput.getPrimaryKey() + " failed with status 404.");
+            }
+            else
+            {
+               QRecord record = apiActionUtil.processSingleRecordGetResponse(table, response);
+               rs.setRecord(record);
+            }
+
             return rs;
          }
       }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java

@@ -76,4 +76,15 @@ public class AbstractFilesystemBackendMetaData extends QBackendMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean requiresPrimaryKeyOnTables()
+   {
+      return (false);
+   }
+
 }