CE-1328 Add job status messages

.circleci/adjust-pom-version.sh

@@ -7,8 +7,6 @@ fi
 
 if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ] || [ \! -z $(echo "$CIRCLE_TAG" | grep "^version-") ]; then
    echo "On a primary branch or tag [${CIRCLE_BRANCH}${CIRCLE_TAG}] - will not edit the pom version.";
-   echo "(or do we need to do this for dev...?)"
-   echo "(maybe we do this if the current version is a SNAPSHOT?)"
    exit 0;
 fi
 
@@ -19,9 +17,7 @@ else
 fi
 
 POM=$(dirname $0)/../pom.xml
-UNIQ=$(date +%Y%m%d-%H%M%S)-$(git rev-parse --short HEAD)
-SLUG="${SLUG}-${UNIQ}"
 
-echo "Updating $POM <revision> to: $SLUG"
-sed -i "s/<revision>.*/<revision>$SLUG<\/revision>/" $POM
+echo "Updating $POM <revision> to: $SLUG-SNAPSHOT"
+sed -i "s/<revision>.*/<revision>$SLUG-SNAPSHOT<\/revision>/" $POM
 git diff $POM

README.md

@@ -48,5 +48,3 @@ GNU Affero General Public License for more details.
 
 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -53,8 +53,6 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    private String  variantOptionsTableUsernameField;
    private String  variantOptionsTablePasswordField;
    private String  variantOptionsTableApiKeyField;
-   private String  variantOptionsTableClientIdField;
-   private String  variantOptionsTableClientSecretField;
    private String  variantOptionsTableName;
 
    // todo - at some point, we may want to apply this to secret properties on subclasses?
@@ -650,66 +648,4 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    {
       qInstance.addBackend(this);
    }
-
-   /*******************************************************************************
-    ** Getter for variantOptionsTableClientIdField
-    *******************************************************************************/
-   public String getVariantOptionsTableClientIdField()
-   {
-      return (this.variantOptionsTableClientIdField);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for variantOptionsTableClientIdField
-    *******************************************************************************/
-   public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
-   {
-      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for variantOptionsTableClientIdField
-    *******************************************************************************/
-   public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
-   {
-      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for variantOptionsTableClientSecretField
-    *******************************************************************************/
-   public String getVariantOptionsTableClientSecretField()
-   {
-      return (this.variantOptionsTableClientSecretField);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for variantOptionsTableClientSecretField
-    *******************************************************************************/
-   public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
-   {
-      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for variantOptionsTableClientSecretField
-    *******************************************************************************/
-   public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
-   {
-      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
-      return (this);
-   }
-
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStep.java

@@ -0,0 +1,287 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class GenericGarbageCollectorExecuteStep implements BackendStep
+{
+   private static final QLogger LOG = QLogger.getLogger(GenericGarbageCollectorExecuteStep.class);
+
+   private ProcessSummaryLine partitionsLine = new ProcessSummaryLine(Status.INFO)
+      .withSingularPastMessage("partition was processed")
+      .withPluralPastMessage("partitions were processed");
+
+   private ProcessSummaryLine deletedLine = new ProcessSummaryLine(Status.OK)
+      .withSingularPastMessage("record was deleted")
+      .withPluralPastMessage("records were deleted");
+
+   private ProcessSummaryLine warningLine = new ProcessSummaryLine(Status.WARNING, "had an warning");
+   private ProcessSummaryLine errorLine   = new ProcessSummaryLine(Status.ERROR, "had an error");
+
+   private AsyncJobCallback asyncJobCallback;
+
+   private Integer total        = null;
+   private Integer deletedSoFar = 0;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      String         tableName = runBackendStepInput.getValueString("table");
+      QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+      if(table == null)
+      {
+         throw new QUserFacingException("Unrecognized table: " + tableName);
+      }
+
+      String         fieldName = runBackendStepInput.getValueString("field");
+      QFieldMetaData field     = table.getFields().get(fieldName);
+      if(field == null)
+      {
+         throw new QUserFacingException("Unrecognized field: " + fieldName);
+      }
+
+      if(!QFieldType.DATE_TIME.equals(field.getType()) && !QFieldType.DATE.equals(field.getType()))
+      {
+         throw new QUserFacingException("Field " + field + " is not a date-time or date type field.");
+      }
+
+      Integer daysBack = runBackendStepInput.getValueInteger("daysBack");
+      if(daysBack == null || daysBack < 0)
+      {
+         throw new QUserFacingException("Illegal value for daysBack: " + daysBack + "; Must be positive.");
+      }
+
+      Integer maxPageSize = runBackendStepInput.getValueInteger("maxPageSize");
+      if(maxPageSize == null || maxPageSize < 0)
+      {
+         throw new QUserFacingException("Illegal value for maxPageSize: " + maxPageSize + "; Must be positive.");
+      }
+
+      asyncJobCallback = runBackendStepInput.getAsyncJobCallback();
+      long start = System.currentTimeMillis();
+      execute(table, field, daysBack, maxPageSize);
+      long end = System.currentTimeMillis();
+
+      deletedLine.prepareForFrontend(true);
+      partitionsLine.prepareForFrontend(true);
+
+      ArrayList<ProcessSummaryLineInterface> processSummary = new ArrayList<>();
+      processSummary.add(partitionsLine);
+      processSummary.add(deletedLine);
+      warningLine.addSelfToListIfAnyCount(processSummary);
+      errorLine.addSelfToListIfAnyCount(processSummary);
+      processSummary.add(new ProcessSummaryLine(Status.INFO, "Total time:  " + String.format("%,d", ((end - start) / 1000)) + " seconds"));
+      runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummary);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void execute(QTableMetaData table, QFieldMetaData field, Integer daysBack, Integer maxPageSize) throws QException
+   {
+      asyncJobCallback.updateStatus("Counting records");
+      Instant maxDate = Instant.now().minusSeconds(daysBack * 60 * 60 * 24);
+      Instant minDate = findMinDateInTable(table, field);
+
+      processDateRange(table, field, maxPageSize, minDate, maxDate);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void processDateRange(QTableMetaData table, QFieldMetaData field, Integer maxPageSize, Instant minDate, Instant maxDate) throws QException
+   {
+      partitionsLine.incrementCount();
+
+      LOG.info("Counting", logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      Integer count = count(table, field, minDate, maxDate);
+      LOG.info("Count", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+
+      if(this.total == null)
+      {
+         this.total = count;
+      }
+
+      if(count == 0)
+      {
+         LOG.info("0 rows in this partition - nothing to delete", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      }
+      else if(count <= maxPageSize)
+      {
+         asyncJobCallback.updateStatus("Deleting records", deletedSoFar, total);
+         LOG.info("Deleting", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+         delete(table, field, minDate, maxDate);
+         this.deletedSoFar += count;
+      }
+      else
+      {
+         if(deletedSoFar == 0)
+         {
+            asyncJobCallback.updateStatus("Partitioning table", deletedSoFar, total);
+         }
+
+         LOG.info("Too many rows", logPair("count", count), logPair("maxPageSize", maxPageSize), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+         partition(table, field, minDate, maxDate, count, maxPageSize);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void partition(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate, Integer count, Integer maxPageSize) throws QException
+   {
+      int  noOfPartitions    = (int) Math.ceil((float) count / (float) maxPageSize);
+      long milliDiff         = maxDate.toEpochMilli() - minDate.toEpochMilli();
+      long milliPerPartition = milliDiff / noOfPartitions;
+
+      if(milliPerPartition < 1000)
+      {
+         throw (new QUserFacingException("To find a maxPageSize under " + String.format("%,d", maxPageSize) + ", the partition size would become smaller than 1 second (between " + minDate + " and " + maxDate + " there are " + String.format("%,d", count) + " rows) - you must use a larger maxPageSize to continue."));
+      }
+
+      LOG.info("Partitioning", logPair("count", count), logPair("noOfPartitions", noOfPartitions), logPair("milliDiff", milliDiff), logPair("milliPerPartition", milliPerPartition), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      for(int i = 0; i < noOfPartitions; i++)
+      {
+         maxDate = minDate.plusMillis(milliPerPartition);
+         processDateRange(table, field, maxPageSize, minDate, maxDate);
+         minDate = maxDate;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void delete(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
+   {
+      DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(table.getName())
+         .withQueryFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate))));
+
+      deletedLine.incrementCount(deleteOutput.getDeletedRecordCount());
+
+      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
+      {
+         warningLine.incrementCount(deleteOutput.getRecordsWithWarnings().size());
+      }
+
+      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
+      {
+         errorLine.incrementCount(deleteOutput.getRecordsWithErrors().size());
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Integer count(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
+   {
+      Aggregate      countDateField = new Aggregate(field.getName(), AggregateOperator.COUNT).withFieldType(QFieldType.INTEGER);
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(table.getName());
+      aggregateInput.withFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate)));
+      aggregateInput.withAggregate(countDateField);
+
+      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+      List<AggregateResult> results         = aggregateOutput.getResults();
+      if(CollectionUtils.nullSafeIsEmpty(results))
+      {
+         throw (new QUserFacingException("Could not count rows table (null or empty aggregate result)."));
+      }
+
+      return (ValueUtils.getValueAsInteger(results.get(0).getAggregateValue(countDateField)));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant findMinDateInTable(QTableMetaData table, QFieldMetaData field) throws QException
+   {
+      Aggregate      minDate        = new Aggregate(field.getName(), AggregateOperator.MIN);
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(table.getName());
+      aggregateInput.withAggregate(minDate);
+
+      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+      List<AggregateResult> results         = aggregateOutput.getResults();
+      if(CollectionUtils.nullSafeIsEmpty(results))
+      {
+         throw (new QUserFacingException("Could not find min date value in table (null or empty aggregate result)."));
+      }
+
+      return (ValueUtils.getValueAsInstant(results.get(0).getAggregateValue(minDate)));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorProcessMetaDataProducer.java

@@ -0,0 +1,88 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** Generic process that can perform garbage collection on any table (at least,
+ ** any table with a date or date-time field).
+ **
+ ** When running, this process prompts for:
+ ** - table name
+ ** - field name (e.g., the date/date-time field on that table)
+ ** - daysBack - any records older than that many days ago will be deleted.
+ ** - maxPageSize - to avoid running "1 huge query", if there are more than
+ ** this number of records between the min-date in the table and the max-date
+ ** (based on daysBack), then the time range is partitioned recursively until
+ ** pages smaller than this parameter are found.  The partitioning attempts to
+ ** be smart (e.g., not just ÷ 2), by doing count / maxPageSize.
+ *******************************************************************************/
+public class GenericGarbageCollectorProcessMetaDataProducer extends MetaDataProducer<QProcessMetaData>
+{
+   public static final String NAME = "GenericGarbageCollector";
+
+
+
+   /*******************************************************************************
+    ** See class header for param descriptions.
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      QProcessMetaData processMetaData = new QProcessMetaData()
+         .withName(NAME)
+         .withIcon(new QIcon().withName("auto_delete"))
+         .withStepList(List.of(
+            new QFrontendStepMetaData()
+               .withName("input")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+               .withFormField(new QFieldMetaData("table", QFieldType.STRING))
+               .withFormField(new QFieldMetaData("field", QFieldType.STRING))
+               .withFormField(new QFieldMetaData("daysBack", QFieldType.INTEGER).withDefaultValue(90))
+               .withFormField(new QFieldMetaData("maxPageSize", QFieldType.INTEGER).withDefaultValue(100000)),
+            new QBackendStepMetaData()
+               .withName("execute")
+               .withCode(new QCodeReference(GenericGarbageCollectorExecuteStep.class)),
+            new QFrontendStepMetaData()
+               .withName("result")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS))
+         ));
+
+      return (processMetaData);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStepTest.java

@@ -0,0 +1,243 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for GenericGarbageCollectorExecuteStep 
+ *******************************************************************************/
+class GenericGarbageCollectorExecuteStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testErrors() throws Exception
+   {
+      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: null");
+
+      input.addValue("table", "notATable");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: notATable");
+
+      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: null");
+
+      input.addValue("field", "notAField");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: notAField");
+
+      input.addValue("field", "firstName");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("not a date");
+
+      input.addValue("field", "timestamp");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: null");
+
+      input.addValue("daysBack", "-1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: -1");
+
+      input.addValue("daysBack", "1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: null");
+
+      input.addValue("maxPageSize", "-1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: -1");
+
+      input.addValue("maxPageSize", "1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Could not find min date value in table");
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void runAndThrow(RunProcessInput input) throws Exception
+   {
+      input.setStartAfterStep(null);
+      input.setProcessUUID(null);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+      if(runProcessOutput.getException().isPresent())
+      {
+         throw (runProcessOutput.getException().get());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test30days() throws QException
+   {
+      insertAndRunGC(30);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(2, queryOutput.getRecords().size());
+      assertEquals(Set.of(4, 5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   @Disabled("memory aggregator is failing to return an aggregate when no rows found, which is throwing an error...")
+   void test100days() throws QException
+   {
+      insertAndRunGC(100);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(5, queryOutput.getRecords().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test10days() throws QException
+   {
+      insertAndRunGC(10);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(1, queryOutput.getRecords().size());
+      assertEquals(Set.of(5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test1day() throws QException
+   {
+      insertAndRunGC(1);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(0, queryOutput.getRecords().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test1dayPartitioned() throws QException
+   {
+      insertAndRunGC(1, 2);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(0, queryOutput.getRecords().size());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void insertAndRunGC(Integer daysBack) throws QException
+   {
+      insertAndRunGC(daysBack, 1000);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void insertAndRunGC(Integer daysBack, Integer maxPageSize) throws QException
+   {
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords()));
+      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
+      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      input.addValue("field", "timestamp");
+      input.addValue("daysBack", daysBack);
+      input.addValue("maxPageSize", maxPageSize);
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+      new RunProcessAction().execute(input);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QRecord> getPersonRecords()
+   {
+      List<QRecord> records = List.of(
+         new QRecord().withValue("id", 1).withValue("timestamp", Instant.now().minus(90, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 2).withValue("timestamp", Instant.now().minus(31, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 3).withValue("timestamp", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
+         new QRecord().withValue("id", 4).withValue("timestamp", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
+         new QRecord().withValue("id", 5).withValue("timestamp", Instant.now().minus(5, ChronoUnit.DAYS)));
+      return records;
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedviews/SavedViewProcessTests.java

@@ -192,6 +192,7 @@ class SavedViewProcessTests extends BaseTest
     **
     *******************************************************************************/
    @Test
+   @SuppressWarnings("unchecked")
    void testNotFoundThrowsProperly() throws QException
    {
       QInstance qInstance = QContext.getQInstance();
@@ -244,4 +245,4 @@ class SavedViewProcessTests extends BaseTest
       }
    }
 
-}
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -35,7 +35,6 @@ import java.util.Base64;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -65,7 +64,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
-import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -89,7 +87,6 @@ import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPatch;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
@@ -121,36 +118,10 @@ public class BaseAPIActionUtil
 
 
    /***************************************************************************
-    ** enum of which HTTP Method the backend uses for Updates.
+    **
     ***************************************************************************/
    public enum UpdateHttpMethod
-   {
-      PUT(HttpPut::new),
-      POST(HttpPost::new),
-      PATCH(HttpPatch::new);
-
-      private Supplier<HttpEntityEnclosingRequestBase> httpEntitySupplier;
-
-
-
-      /***************************************************************************
-       **
-       ***************************************************************************/
-      UpdateHttpMethod(Supplier<HttpEntityEnclosingRequestBase> httpEnttySupplier)
-      {
-         this.httpEntitySupplier = httpEnttySupplier;
-      }
-
-
-
-      /***************************************************************************
-       **
-       ***************************************************************************/
-      public HttpEntityEnclosingRequestBase newRequest()
-      {
-         return (this.httpEntitySupplier.get());
-      }
-   }
+   {PUT, POST}
 
 
 
@@ -379,9 +350,7 @@ public class BaseAPIActionUtil
             {
                String                         paramString = buildQueryStringForUpdate(table, recordList);
                String                         url         = buildTableUrl(table) + paramString;
-               HttpEntityEnclosingRequestBase request     = getUpdateMethod().newRequest();
-
-               request.setURI(new URI(url));
+               HttpEntityEnclosingRequestBase request     = getUpdateMethod().equals(UpdateHttpMethod.PUT) ? new HttpPut(url) : new HttpPost(url);
                request.setEntity(recordsToEntity(table, recordList));
 
                QHttpResponse response = makeRequest(table, request);
@@ -719,22 +688,60 @@ public class BaseAPIActionUtil
     *******************************************************************************/
    public void setupAuthorizationInRequest(HttpRequestBase request) throws QException
    {
-      ///////////////////////////////////////////////////////////////////
-      // update the request based on the authorization type being used //
-      ///////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////////////////////
+      // if backend specifies that it uses variants, look for that data in the session //
+      ///////////////////////////////////////////////////////////////////////////////////
+      if(backendMetaData.getUsesVariants())
+      {
+         QSession session = QContext.getQSession();
+         if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
+         {
+            throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"));
+         }
+
+         Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
+         GetInput     getInput  = new GetInput();
+         getInput.setShouldMaskPasswords(false);
+         getInput.setTableName(backendMetaData.getVariantOptionsTableName());
+         getInput.setPrimaryKey(variantId);
+         GetOutput getOutput = new GetAction().execute(getInput);
+
+         QRecord record = getOutput.getRecord();
+         if(record == null)
+         {
+            throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'"));
+         }
+
+         if(backendMetaData.getAuthorizationType().equals(AuthorizationType.BASIC_AUTH_USERNAME_PASSWORD))
+         {
+            request.setHeader("Authorization", getBasicAuthenticationHeader(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
+         }
+         else if(backendMetaData.getAuthorizationType().equals(AuthorizationType.API_KEY_HEADER))
+         {
+            request.setHeader("API-Key", record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
+         }
+         else
+         {
+            throw (new IllegalArgumentException("Unexpected variant authorization type specified: " + backendMetaData.getAuthorizationType()));
+         }
+         return;
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // if not using variants, the authorization data will be in the backend meta data object //
+      ///////////////////////////////////////////////////////////////////////////////////////////
       switch(backendMetaData.getAuthorizationType())
       {
-         case BASIC_AUTH_API_KEY -> request.setHeader("Authorization", getBasicAuthenticationHeader(getApiKey()));
-         case BASIC_AUTH_USERNAME_PASSWORD ->
-         {
-            Pair<String, String> usernameAndPassword = getUsernameAndPassword();
-            request.setHeader("Authorization", getBasicAuthenticationHeader(usernameAndPassword.getA(), usernameAndPassword.getB()));
-         }
-         case API_KEY_HEADER -> request.setHeader("API-Key", getApiKey());
-         case API_TOKEN -> request.setHeader("Authorization", "Token " + getApiKey());
+         case BASIC_AUTH_API_KEY -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getApiKey()));
+         case BASIC_AUTH_USERNAME_PASSWORD -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getUsername(), backendMetaData.getPassword()));
+         case API_KEY_HEADER -> request.setHeader("API-Key", backendMetaData.getApiKey());
+         case API_TOKEN -> request.setHeader("Authorization", "Token " + backendMetaData.getApiKey());
          case OAUTH2 -> request.setHeader("Authorization", "Bearer " + getOAuth2Token());
          case API_KEY_QUERY_PARAM -> addApiKeyQueryParamToRequest(request);
-         case CUSTOM -> handleCustomAuthorization(request);
+         case CUSTOM ->
+         {
+            handleCustomAuthorization(request);
+         }
          default -> throw new IllegalArgumentException("Unexpected authorization type: " + backendMetaData.getAuthorizationType());
       }
    }
@@ -749,7 +756,7 @@ public class BaseAPIActionUtil
       try
       {
          String uri = request.getURI().toString();
-         String pair = backendMetaData.getApiKeyQueryParamName() + "=" + getApiKey();
+         String pair = backendMetaData.getApiKeyQueryParamName() + "=" + backendMetaData.getApiKey();
 
          ///////////////////////////////////////////////////////////////////////////////////
          // avoid re-adding the name=value pair if it's already there (e.g., for a retry) //
@@ -770,106 +777,39 @@ public class BaseAPIActionUtil
 
 
 
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   protected String getApiKey() throws QException
-   {
-      if(backendMetaData.getUsesVariants())
-      {
-         QRecord record = getVariantRecord();
-         return (record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
-      }
-
-      return (backendMetaData.getApiKey());
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   protected Pair<String, String> getUsernameAndPassword() throws QException
-   {
-      if(backendMetaData.getUsesVariants())
-      {
-         QRecord record = getVariantRecord();
-         return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
-      }
-
-      return (Pair.of(backendMetaData.getUsername(), backendMetaData.getPassword()));
-   }
-
-
-
-   /*******************************************************************************
-    ** For backends that use variants, look up the variant record (in theory, based
-    ** on an id in the session's backend variants map, then fetched from the backend's
-    ** variant options table.
-    *******************************************************************************/
-   protected QRecord getVariantRecord() throws QException
-   {
-      Serializable variantId = getVariantId();
-      GetInput     getInput  = new GetInput();
-      getInput.setShouldMaskPasswords(false);
-      getInput.setTableName(backendMetaData.getVariantOptionsTableName());
-      getInput.setPrimaryKey(variantId);
-      GetOutput getOutput = new GetAction().execute(getInput);
-
-      QRecord record = getOutput.getRecord();
-      if(record == null)
-      {
-         throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'"));
-      }
-      return record;
-   }
-
-
-
-   /*******************************************************************************
-    ** Get the variant id from the session for the backend.
-    *******************************************************************************/
-   protected Serializable getVariantId() throws QException
-   {
-      QSession session = QContext.getQSession();
-      if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
-      {
-         throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"));
-      }
-      Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
-      return variantId;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
-   public String getOAuth2Token() throws OAuthCredentialsException, QException
+   public String getOAuth2Token() throws OAuthCredentialsException
    {
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // define the key that will be used in the backend's customValues map, to stash the access token.                   //
-      // for non-variant backends, this is just a constant string.  But for variant-backends, append the variantId to it. //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      String accessTokenKey = "accessToken";
-      if(backendMetaData.getUsesVariants())
-      {
-         Serializable variantId = getVariantId();
-         accessTokenKey = accessTokenKey + ":" + variantId;
-      }
-
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // check for the access token in the backend meta data.  if it's not there, then issue a request for a token. //
       // this is not generally meant to be put in the meta data by the app programmer - rather, we're just using    //
       // it as a "cheap & easy" way to "cache" the token within our process's memory...                             //
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      String accessToken = ValueUtils.getValueAsString(backendMetaData.getCustomValue(accessTokenKey));
+      String  accessToken            = ValueUtils.getValueAsString(backendMetaData.getCustomValue("accessToken"));
+      Boolean setCredentialsInHeader = BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(backendMetaData.getCustomValue("setCredentialsInHeader")));
 
       if(!StringUtils.hasContent(accessToken))
       {
+         String fullURL  = backendMetaData.getBaseUrl() + "oauth/token";
+         String postBody = "grant_type=client_credentials";
+
+         if(!setCredentialsInHeader)
+         {
+            postBody += "&client_id=" + backendMetaData.getClientId() + "&client_secret=" + backendMetaData.getClientSecret();
+         }
+
          try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(new PoolingHttpClientConnectionManager()).build())
          {
-            HttpRequestBase request = createOAuth2TokenRequest();
+            HttpPost request = new HttpPost(fullURL);
+            request.setEntity(new StringEntity(postBody, getCharsetForEntity()));
+
+            if(setCredentialsInHeader)
+            {
+               request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
+            }
+            request.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8");
 
             HttpResponse response     = executeOAuthTokenRequest(client, request);
             int          statusCode   = response.getStatusLine().getStatusCode();
@@ -887,7 +827,7 @@ public class BaseAPIActionUtil
             ///////////////////////////////////////////////////////////////////////////////////////////////////
             // stash the access token in the backendMetaData, from which it will be used for future requests //
             ///////////////////////////////////////////////////////////////////////////////////////////////////
-            backendMetaData.withCustomValue(accessTokenKey, accessToken);
+            backendMetaData.withCustomValue("accessToken", accessToken);
          }
          catch(OAuthCredentialsException oce)
          {
@@ -906,53 +846,6 @@ public class BaseAPIActionUtil
 
 
 
-   /***************************************************************************
-    ** For doing OAuth2 authentication, create a request for a token.
-    ***************************************************************************/
-   protected HttpRequestBase createOAuth2TokenRequest() throws QException
-   {
-      String fullURL  = backendMetaData.getBaseUrl() + "oauth/token";
-      String postBody = "grant_type=client_credentials";
-
-      Pair<String, String> clientIdAndSecret = getClientIdAndSecret();
-      String               clientId          = clientIdAndSecret.getA();
-      String               clientSecret      = clientIdAndSecret.getB();
-
-      Boolean setCredentialsInHeader = BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(backendMetaData.getCustomValue("setCredentialsInHeader")));
-      if(!setCredentialsInHeader)
-      {
-         postBody += "&client_id=" + clientId + "&client_secret=" + clientSecret;
-      }
-
-      HttpPost request = new HttpPost(fullURL);
-      request.setEntity(new StringEntity(postBody, getCharsetForEntity()));
-
-      if(setCredentialsInHeader)
-      {
-         request.setHeader("Authorization", getBasicAuthenticationHeader(clientId, clientSecret));
-      }
-      request.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8");
-      return request;
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   protected Pair<String, String> getClientIdAndSecret() throws QException
-   {
-      if(backendMetaData.getUsesVariants())
-      {
-         QRecord record = getVariantRecord();
-         return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableClientIdField()), record.getValueString(backendMetaData.getVariantOptionsTableClientSecretField())));
-      }
-
-      return (Pair.of(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
-   }
-
-
-
    /*******************************************************************************
     ** Let a subclass change what charset to use for entities (bodies) being posted/put/etc.
     *******************************************************************************/
@@ -966,18 +859,6 @@ public class BaseAPIActionUtil
    /*******************************************************************************
     ** one-line method, factored out so mock/tests can override
     *******************************************************************************/
-   protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpRequestBase request) throws IOException
-   {
-      return client.execute(request);
-   }
-
-
-
-   /*******************************************************************************
-    ** one-line method, factored out so mock/tests can override
-    ** Deprecated, in favor of more generic overload that takes HttpRequestBase
-    *******************************************************************************/
-   @Deprecated
    protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpPost request) throws IOException
    {
       return client.execute(request);

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogHeader.java

@@ -0,0 +1,316 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model;
+
+
+import java.time.Instant;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QAssociation;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for OutboundApiLog table
+ *******************************************************************************/
+public class OutboundAPILogHeader extends QRecordEntity
+{
+   public static final String TABLE_NAME = "outboundApiLogHeader";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField()
+   private Instant timestamp;
+
+   @QField(possibleValueSourceName = "outboundApiMethod")
+   private String method;
+
+   @QField(possibleValueSourceName = "outboundApiStatusCode")
+   private Integer statusCode;
+
+   @QField(label = "URL")
+   private String url;
+
+   @QAssociation(name = OutboundAPILogRequest.TABLE_NAME)
+   private List<OutboundAPILogRequest> outboundAPILogRequestList;
+
+   @QAssociation(name = OutboundAPILogResponse.TABLE_NAME)
+   private List<OutboundAPILogResponse> outboundAPILogResponseList;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    **
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for timestamp
+    **
+    *******************************************************************************/
+   public Instant getTimestamp()
+   {
+      return timestamp;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for timestamp
+    **
+    *******************************************************************************/
+   public void setTimestamp(Instant timestamp)
+   {
+      this.timestamp = timestamp;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for timestamp
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withTimestamp(Instant timestamp)
+   {
+      this.timestamp = timestamp;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for method
+    **
+    *******************************************************************************/
+   public String getMethod()
+   {
+      return method;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for method
+    **
+    *******************************************************************************/
+   public void setMethod(String method)
+   {
+      this.method = method;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for method
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withMethod(String method)
+   {
+      this.method = method;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for statusCode
+    **
+    *******************************************************************************/
+   public Integer getStatusCode()
+   {
+      return statusCode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for statusCode
+    **
+    *******************************************************************************/
+   public void setStatusCode(Integer statusCode)
+   {
+      this.statusCode = statusCode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for statusCode
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withStatusCode(Integer statusCode)
+   {
+      this.statusCode = statusCode;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for url
+    **
+    *******************************************************************************/
+   public String getUrl()
+   {
+      return url;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for url
+    **
+    *******************************************************************************/
+   public void setUrl(String url)
+   {
+      this.url = url;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for url
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withUrl(String url)
+   {
+      this.url = url;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundAPILogRequestList
+    *******************************************************************************/
+   public List<OutboundAPILogRequest> getOutboundAPILogRequestList()
+   {
+      return (this.outboundAPILogRequestList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundAPILogRequestList
+    *******************************************************************************/
+   public void setOutboundAPILogRequestList(List<OutboundAPILogRequest> outboundAPILogRequestList)
+   {
+      this.outboundAPILogRequestList = outboundAPILogRequestList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundAPILogRequestList
+    *******************************************************************************/
+   public OutboundAPILogHeader withOutboundAPILogRequestList(List<OutboundAPILogRequest> outboundAPILogRequestList)
+   {
+      this.outboundAPILogRequestList = outboundAPILogRequestList;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundAPILogResponseList
+    *******************************************************************************/
+   public List<OutboundAPILogResponse> getOutboundAPILogResponseList()
+   {
+      return (this.outboundAPILogResponseList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundAPILogResponseList
+    *******************************************************************************/
+   public void setOutboundAPILogResponseList(List<OutboundAPILogResponse> outboundAPILogResponseList)
+   {
+      this.outboundAPILogResponseList = outboundAPILogResponseList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundAPILogResponseList
+    *******************************************************************************/
+   public OutboundAPILogHeader withOutboundAPILogResponseList(List<OutboundAPILogResponse> outboundAPILogResponseList)
+   {
+      this.outboundAPILogResponseList = outboundAPILogResponseList;
+      return (this);
+   }
+
+}
+

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogMetaDataProvider.java

@@ -22,21 +22,33 @@
 package com.kingsrook.qqq.backend.module.api.model;
 
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogExtractStep;
+import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogLoadStep;
+import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogTransformStep;
 
 
 /*******************************************************************************
@@ -50,8 +62,15 @@ public class OutboundAPILogMetaDataProvider
     *******************************************************************************/
    public static void defineAll(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      definePossibleValueSources(qInstance);
-      defineOutboundAPILogTable(qInstance, backendName, backendDetailEnricher);
+      definePossibleValueSources().forEach(pvs ->
+      {
+         if(qInstance.getPossibleValueSource(pvs.getName()) == null)
+         {
+            qInstance.addPossibleValueSource(pvs);
+         }
+      });
+
+      qInstance.addTable(defineOutboundAPILogTable(backendName, backendDetailEnricher));
    }
 
 
@@ -59,9 +78,71 @@ public class OutboundAPILogMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void definePossibleValueSources(QInstance instance)
+   public static void defineNewVersion(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      instance.addPossibleValueSource(new QPossibleValueSource()
+      definePossibleValueSources().forEach(pvs ->
+      {
+         if(qInstance.getPossibleValueSource(pvs.getName()) == null)
+         {
+            qInstance.addPossibleValueSource(pvs);
+         }
+      });
+
+      qInstance.addTable(defineOutboundAPILogHeaderTable(backendName, backendDetailEnricher));
+      qInstance.addPossibleValueSource(defineOutboundAPILogHeaderPossibleValueSource());
+      qInstance.addTable(defineOutboundAPILogRequestTable(backendName, backendDetailEnricher));
+      qInstance.addTable(defineOutboundAPILogResponseTable(backendName, backendDetailEnricher));
+      defineJoins().forEach(join -> qInstance.add(join));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static void defineMigrationProcesses(QInstance qInstance, String sourceTableName)
+   {
+      qInstance.addProcess(StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName("migrateOutboundApiLogToHeaderChildProcess")
+         .withLabel("Migrate Outbound API Log Test to Header/Child")
+         .withIcon(new QIcon().withName("drive_file_move"))
+         .withTableName(sourceTableName)
+         .withSourceTable(sourceTableName)
+         .withDestinationTable(OutboundAPILogHeader.TABLE_NAME)
+         .withExtractStepClass(MigrateOutboundAPILogExtractStep.class)
+         .withTransformStepClass(MigrateOutboundAPILogTransformStep.class)
+         .withLoadStepClass(MigrateOutboundAPILogLoadStep.class)
+         .withReviewStepRecordFields(List.of(
+            new QFieldMetaData("url", QFieldType.INTEGER)
+         ))
+         .getProcessMetaData());
+
+      qInstance.addProcess(StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName("migrateOutboundApiLogToMongoDBProcess")
+         .withLabel("Migrate Outbound API Log Test to MongoDB")
+         .withIcon(new QIcon().withName("drive_file_move"))
+         .withTableName(sourceTableName)
+         .withSourceTable(sourceTableName)
+         .withDestinationTable(OutboundAPILog.TABLE_NAME + "MongoDB")
+         .withExtractStepClass(MigrateOutboundAPILogExtractStep.class)
+         .withTransformStepClass(MigrateOutboundAPILogTransformStep.class)
+         .withLoadStepClass(MigrateOutboundAPILogLoadStep.class)
+         .withReviewStepRecordFields(List.of(
+            new QFieldMetaData("url", QFieldType.INTEGER)
+         ))
+         .getProcessMetaData());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QPossibleValueSource> definePossibleValueSources()
+   {
+      List<QPossibleValueSource> rs = new ArrayList<>();
+
+      rs.add(new QPossibleValueSource()
          .withName("outboundApiMethod")
          .withType(QPossibleValueSourceType.ENUM)
          .withEnumValues(List.of(
@@ -72,7 +153,7 @@ public class OutboundAPILogMetaDataProvider
             new QPossibleValue<>("DELETE")
          )));
 
-      instance.addPossibleValueSource(new QPossibleValueSource()
+      rs.add(new QPossibleValueSource()
          .withName("outboundApiStatusCode")
          .withType(QPossibleValueSourceType.ENUM)
          .withEnumValues(List.of(
@@ -91,6 +172,8 @@ public class OutboundAPILogMetaDataProvider
             new QPossibleValue<>(503, "503 (Service Unavailable)"),
             new QPossibleValue<>(504, "500 (Gateway Timeout)")
          )));
+
+      return (rs);
    }
 
 
@@ -98,9 +181,8 @@ public class OutboundAPILogMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void defineOutboundAPILogTable(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   public static QTableMetaData defineOutboundAPILogTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-
       QTableMetaData tableMetaData = new QTableMetaData()
          .withName(OutboundAPILog.TABLE_NAME)
          .withLabel("Outbound API Log")
@@ -119,29 +201,8 @@ public class OutboundAPILogMetaDataProvider
       tableMetaData.getField("requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
       tableMetaData.getField("responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
 
-      tableMetaData.getField("method").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
-         .withValue(AdornmentType.ChipValues.colorValue("GET", AdornmentType.ChipValues.COLOR_INFO))
-         .withValue(AdornmentType.ChipValues.colorValue("POST", AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue("DELETE", AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue("PATCH", AdornmentType.ChipValues.COLOR_WARNING))
-         .withValue(AdornmentType.ChipValues.colorValue("PUT", AdornmentType.ChipValues.COLOR_WARNING)));
-
-      tableMetaData.getField("statusCode").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
-         .withValue(AdornmentType.ChipValues.colorValue(200, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(201, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(204, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(207, AdornmentType.ChipValues.COLOR_INFO))
-         .withValue(AdornmentType.ChipValues.colorValue(400, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(401, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(403, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(404, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(422, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(429, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(500, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(502, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(503, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(504, AdornmentType.ChipValues.COLOR_ERROR))
-      );
+      addChipAdornmentToMethodField(tableMetaData);
+      addChipAdornmentToStatusCodeField(tableMetaData);
 
       ///////////////////////////////////////////
       // these are the lengths of a MySQL TEXT //
@@ -160,6 +221,210 @@ public class OutboundAPILogMetaDataProvider
          backendDetailEnricher.accept(tableMetaData);
       }
 
-      qInstance.addTable(tableMetaData);
+      return (tableMetaData);
    }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void addChipAdornmentToStatusCodeField(QTableMetaData tableMetaData)
+   {
+      tableMetaData.getField("statusCode").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
+         .withValue(AdornmentType.ChipValues.colorValue(200, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(201, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(204, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(207, AdornmentType.ChipValues.COLOR_INFO))
+         .withValue(AdornmentType.ChipValues.colorValue(400, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(401, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(403, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(404, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(422, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(429, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(500, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(502, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(503, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(504, AdornmentType.ChipValues.COLOR_ERROR))
+      );
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void addChipAdornmentToMethodField(QTableMetaData tableMetaData)
+   {
+      tableMetaData.getField("method").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
+         .withValue(AdornmentType.ChipValues.colorValue("GET", AdornmentType.ChipValues.COLOR_INFO))
+         .withValue(AdornmentType.ChipValues.colorValue("POST", AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue("DELETE", AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue("PATCH", AdornmentType.ChipValues.COLOR_WARNING))
+         .withValue(AdornmentType.ChipValues.colorValue("PUT", AdornmentType.ChipValues.COLOR_WARNING)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineOutboundAPILogHeaderTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(OutboundAPILogHeader.TABLE_NAME)
+         .withLabel("Outbound API Log Header/Child")
+         .withIcon(new QIcon().withName("data_object"))
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(OutboundAPILogHeader.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
+         .withSection(new QFieldSection("request", new QIcon().withName("arrow_upward"), Tier.T2, List.of("method", "url", OutboundAPILogRequest.TABLE_NAME + ".requestBody")))
+         .withSection(new QFieldSection("response", new QIcon().withName("arrow_downward"), Tier.T2, List.of("statusCode", OutboundAPILogResponse.TABLE_NAME + ".responseBody")))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("timestamp")))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+
+      // tableMetaData.getField(OutboundAPILogRequest.TABLE_NAME + ".requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+      // tableMetaData.getField(OutboundAPILogResponse.TABLE_NAME + ".responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+
+      addChipAdornmentToMethodField(tableMetaData);
+      addChipAdornmentToStatusCodeField(tableMetaData);
+
+      tableMetaData.withAssociation(new Association()
+         .withName(OutboundAPILogRequest.TABLE_NAME)
+         .withAssociatedTableName(OutboundAPILogRequest.TABLE_NAME)
+         .withJoinName(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogRequest.TABLE_NAME)));
+
+      tableMetaData.withAssociation(new Association()
+         .withName(OutboundAPILogResponse.TABLE_NAME)
+         .withAssociatedTableName(OutboundAPILogResponse.TABLE_NAME)
+         .withJoinName(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogResponse.TABLE_NAME)));
+
+      tableMetaData.withExposedJoin(new ExposedJoin()
+         .withJoinTable(OutboundAPILogRequest.TABLE_NAME)
+         .withJoinPath(List.of(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogRequest.TABLE_NAME))));
+
+      tableMetaData.withExposedJoin(new ExposedJoin()
+         .withJoinTable(OutboundAPILogResponse.TABLE_NAME)
+         .withJoinPath(List.of(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogResponse.TABLE_NAME))));
+
+      tableMetaData.getField("url").withMaxLength(4096).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+      tableMetaData.getField("url").withFieldAdornment(AdornmentType.Size.XLARGE.toAdornment());
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(tableMetaData);
+      }
+
+      return (tableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineOutboundAPILogRequestTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(OutboundAPILogRequest.TABLE_NAME)
+         .withLabel("Outbound API Log Request")
+         .withIcon(new QIcon().withName("arrow_upward"))
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(OutboundAPILogRequest.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "outboundApiLogHeaderId")))
+         .withSection(new QFieldSection("request", new QIcon().withName("arrow_upward"), Tier.T2, List.of("requestBody")))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+
+      tableMetaData.getField("requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+
+      //////////////////////////////////////////////
+      // this is the length of a MySQL MEDIUMTEXT //
+      //////////////////////////////////////////////
+      tableMetaData.getField("requestBody").withMaxLength(16_777_215).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(tableMetaData);
+      }
+
+      return (tableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineOutboundAPILogResponseTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(OutboundAPILogResponse.TABLE_NAME)
+         .withLabel("Outbound API Log Response")
+         .withIcon(new QIcon().withName("arrow_upward"))
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(OutboundAPILogResponse.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "outboundApiLogHeaderId")))
+         .withSection(new QFieldSection("response", new QIcon().withName("arrow_upward"), Tier.T2, List.of("responseBody")))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+
+      tableMetaData.getField("responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+
+      //////////////////////////////////////////////
+      // this is the length of a MySQL MEDIUMTEXT //
+      //////////////////////////////////////////////
+      tableMetaData.getField("responseBody").withMaxLength(16_777_215).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(tableMetaData);
+      }
+
+      return (tableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QJoinMetaData> defineJoins()
+   {
+      List<QJoinMetaData> rs = new ArrayList<>();
+
+      rs.add(new QJoinMetaData()
+         .withLeftTable(OutboundAPILogHeader.TABLE_NAME)
+         .withRightTable(OutboundAPILogRequest.TABLE_NAME)
+         .withInferredName()
+         .withType(JoinType.ONE_TO_ONE)
+         .withJoinOn(new JoinOn("id", "outboundApiLogHeaderId")));
+
+      rs.add(new QJoinMetaData()
+         .withLeftTable(OutboundAPILogHeader.TABLE_NAME)
+         .withRightTable(OutboundAPILogResponse.TABLE_NAME)
+         .withInferredName()
+         .withType(JoinType.ONE_TO_ONE)
+         .withJoinOn(new JoinOn("id", "outboundApiLogHeaderId")));
+
+      return (rs);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static QPossibleValueSource defineOutboundAPILogHeaderPossibleValueSource()
+   {
+      return QPossibleValueSource.newForTable(OutboundAPILogHeader.TABLE_NAME);
+   }
+
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogRequest.java

@@ -0,0 +1,165 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for OutboundApiLogRequest table
+ *******************************************************************************/
+public class OutboundAPILogRequest extends QRecordEntity
+{
+   public static final String TABLE_NAME = "outboundApiLogRequest";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(possibleValueSourceName = OutboundAPILogHeader.TABLE_NAME)
+   private Integer outboundApiLogHeaderId;
+
+   @QField()
+   private String requestBody;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogRequest()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogRequest(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    **
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    **
+    *******************************************************************************/
+   public OutboundAPILogRequest withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for requestBody
+    *******************************************************************************/
+   public String getRequestBody()
+   {
+      return (this.requestBody);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for requestBody
+    *******************************************************************************/
+   public void setRequestBody(String requestBody)
+   {
+      this.requestBody = requestBody;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for requestBody
+    *******************************************************************************/
+   public OutboundAPILogRequest withRequestBody(String requestBody)
+   {
+      this.requestBody = requestBody;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public Integer getOutboundApiLogHeaderId()
+   {
+      return (this.outboundApiLogHeaderId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public void setOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public OutboundAPILogRequest withOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+      return (this);
+   }
+
+}
+

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogResponse.java

@@ -0,0 +1,165 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for OutboundApiLogResponse table
+ *******************************************************************************/
+public class OutboundAPILogResponse extends QRecordEntity
+{
+   public static final String TABLE_NAME = "outboundApiLogResponse";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(possibleValueSourceName = OutboundAPILogHeader.TABLE_NAME)
+   private Integer outboundApiLogHeaderId;
+
+   @QField()
+   private String responseBody;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogResponse()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogResponse(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    **
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    **
+    *******************************************************************************/
+   public OutboundAPILogResponse withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for responseBody
+    *******************************************************************************/
+   public String getResponseBody()
+   {
+      return (this.responseBody);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for responseBody
+    *******************************************************************************/
+   public void setResponseBody(String responseBody)
+   {
+      this.responseBody = responseBody;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for responseBody
+    *******************************************************************************/
+   public OutboundAPILogResponse withResponseBody(String responseBody)
+   {
+      this.responseBody = responseBody;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public Integer getOutboundApiLogHeaderId()
+   {
+      return (this.outboundApiLogHeaderId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public void setOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public OutboundAPILogResponse withOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+      return (this);
+   }
+
+}
+

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogExtractStep.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MigrateOutboundAPILogExtractStep extends ExtractViaQueryStep
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected void customizeInputPreQuery(QueryInput queryInput)
+   {
+      queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
+   }
+
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogLoadStep.java

@@ -0,0 +1,54 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
+
+
+/*******************************************************************************
+ ** store outboundApiLogHeaders and maybe more...
+ *******************************************************************************/
+public class MigrateOutboundAPILogLoadStep extends LoadViaInsertStep
+{
+   private static final QLogger LOG = QLogger.getLogger(MigrateOutboundAPILogLoadStep.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      super.runOnePage(runBackendStepInput, runBackendStepOutput);
+
+      ///////////////////////////////////////
+      // todo - track what we've migrated? //
+      ///////////////////////////////////////
+   }
+
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogTransformStep.java

@@ -0,0 +1,127 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogHeader;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogRequest;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogResponse;
+
+
+/*******************************************************************************
+ ** migrate records from original (singular) outboundApiLog table to new split-up
+ ** version (outboundApiLogHeader)
+ *******************************************************************************/
+public class MigrateOutboundAPILogTransformStep extends AbstractTransformStep
+{
+   private static final QLogger LOG = QLogger.getLogger(MigrateOutboundAPILogTransformStep.class);
+
+   private ProcessSummaryLine okToInsertLine = StandardProcessSummaryLineProducer.getOkToInsertLine();
+   private ProcessSummaryLine errorLine      = StandardProcessSummaryLineProducer.getErrorLine();
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   /*
+   @Override
+   public Integer getOverrideRecordPipeCapacity(RunBackendStepInput runBackendStepInput)
+   {
+      return (100);
+   }
+   */
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
+   {
+      ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
+      okToInsertLine.addSelfToListIfAnyCount(rs);
+      errorLine.addSelfToListIfAnyCount(rs);
+      return (rs);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      runBackendStepOutput.addValue("counter", 0);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      int counter = runBackendStepOutput.getValueInteger("counter") + runBackendStepInput.getRecords().size();
+      runBackendStepOutput.addValue("counter", counter);
+      runBackendStepInput.getAsyncJobCallback().updateStatus("Migrating records (at #" + String.format("%,d", counter) + ")");
+
+      String destinationTable = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE);
+
+      for(QRecord record : runBackendStepInput.getRecords())
+      {
+         okToInsertLine.incrementCountAndAddPrimaryKey(record.getValue("id"));
+
+         if(destinationTable.equals(OutboundAPILogHeader.TABLE_NAME))
+         {
+            OutboundAPILogHeader outboundAPILogHeader = new OutboundAPILogHeader(record);
+            outboundAPILogHeader.withOutboundAPILogRequestList(List.of(new OutboundAPILogRequest().withRequestBody(record.getValueString("requestBody"))));
+            outboundAPILogHeader.withOutboundAPILogResponseList(List.of(new OutboundAPILogResponse().withResponseBody(record.getValueString("responseBody"))));
+            runBackendStepOutput.addRecord(outboundAPILogHeader.toQRecord());
+         }
+         else
+         {
+            ///////////////////////////////////////////////////////////////////
+            // for the mongodb migration, just pass records straight through //
+            ///////////////////////////////////////////////////////////////////
+            record.setValue("id", null);
+            runBackendStepOutput.addRecord(record);
+         }
+      }
+   }
+
+}

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/BaseTest.java

@@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 
@@ -46,6 +47,7 @@ public class BaseTest
    void baseBeforeEach()
    {
       QContext.init(TestUtils.defineInstance(), new QSession());
+      MemoryRecordStore.getInstance().reset();
    }
 
 
@@ -57,6 +59,7 @@ public class BaseTest
    void baseAfterEach()
    {
       QContext.clear();
+      MemoryRecordStore.getInstance().reset();
    }
 
 

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/mocks/MockApiActionUtils.java

@@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.module.api.actions.BaseAPIActionUtil;
 import com.kingsrook.qqq.backend.module.api.actions.QHttpResponse;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.impl.client.CloseableHttpClient;
 
@@ -88,7 +89,7 @@ public class MockApiActionUtils extends BaseAPIActionUtil
     **
     *******************************************************************************/
    @Override
-   protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpRequestBase request) throws IOException
+   protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpPost request) throws IOException
    {
       runMockAsserter(request);
       return new MockHttpResponse(mockApiUtilsHelper);

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogProcessTest.java

@@ -0,0 +1,98 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.module.api.BaseTest;
+import com.kingsrook.qqq.backend.module.api.TestUtils;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogHeader;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogMetaDataProvider;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for MigrateOutboundAPILog process
+ *******************************************************************************/
+class MigrateOutboundAPILogProcessTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void beforeEach() throws QException
+   {
+      MemoryRecordStore.getInstance().reset();
+      OutboundAPILogMetaDataProvider.defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
+      OutboundAPILogMetaDataProvider.defineNewVersion(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
+      OutboundAPILogMetaDataProvider.defineMigrationProcesses(QContext.getQInstance(), OutboundAPILog.TABLE_NAME);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      new InsertAction().execute(new InsertInput(OutboundAPILog.TABLE_NAME).withRecordEntity(new OutboundAPILog()
+         .withMethod("POST")
+         .withUrl("www.google.com")
+         .withRequestBody("please")
+         .withResponseBody("you're welcome")
+         .withStatusCode(201)
+      ));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName("migrateOutboundApiLogToHeaderChildProcess");
+      input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter()));
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+
+      List<OutboundAPILogHeader> outboundApiLogHeaderList = new QueryAction().execute(new QueryInput(OutboundAPILogHeader.TABLE_NAME).withIncludeAssociations(true)).getRecordEntities(OutboundAPILogHeader.class);
+      assertEquals(1, outboundApiLogHeaderList.size());
+      assertEquals("POST", outboundApiLogHeaderList.get(0).getMethod());
+      assertEquals(201, outboundApiLogHeaderList.get(0).getStatusCode());
+      assertEquals(1, outboundApiLogHeaderList.get(0).getOutboundAPILogRequestList().size());
+      assertEquals("please", outboundApiLogHeaderList.get(0).getOutboundAPILogRequestList().get(0).getRequestBody());
+      assertEquals(1, outboundApiLogHeaderList.get(0).getOutboundAPILogResponseList().size());
+      assertEquals("you're welcome", outboundApiLogHeaderList.get(0).getOutboundAPILogResponseList().get(0).getResponseBody());
+   }
+
+}

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInputFieldsContainer.java

@@ -43,6 +43,32 @@ public class ApiProcessInputFieldsContainer
 
 
 
+   /***************************************************************************
+    ** factory method to build one of these containers using all of the input fields
+    ** in a process
+    ***************************************************************************/
+   public static ApiProcessInputFieldsContainer forAllInputFields(QProcessMetaData process)
+   {
+      return forFields(process.getInputFields());
+   }
+
+
+
+   /***************************************************************************
+    ** factory method to build one of these containers using a list of fields.
+    ***************************************************************************/
+   public static ApiProcessInputFieldsContainer forFields(List<QFieldMetaData> fields)
+   {
+      ApiProcessInputFieldsContainer container = new ApiProcessInputFieldsContainer();
+      for(QFieldMetaData inputField : CollectionUtils.nonNullList(fields))
+      {
+         container.withField(inputField);
+      }
+      return (container);
+   }
+
+
+
    /*******************************************************************************
     ** find all input fields in frontend steps of the process, and add them as fields
     ** in this container.

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessSummaryListOutput.java

@@ -48,7 +48,9 @@ import org.eclipse.jetty.http.HttpStatus;
 
 
 /*******************************************************************************
- **
+ ** For a process that puts "processResults" in its output (as a list of
+ ** ProcessSummaryLineInterface objects) - this class converts such an object
+ ** to a suitable ApiProcessOutput.
  *******************************************************************************/
 public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
 {
@@ -143,22 +145,31 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
          {
             if(processSummaryLineInterface instanceof ProcessSummaryLine processSummaryLine)
             {
-               processSummaryLine.setCount(1);
-               processSummaryLine.pickMessage(true);
-
                List<Serializable> primaryKeys = processSummaryLine.getPrimaryKeys();
                if(CollectionUtils.nullSafeHasContents(primaryKeys))
                {
+                  ////////////////////////////////////////////////////////////////////////////
+                  // if there are primary keys in the line, then we'll loop over those, and //
+                  // output an object in the API output for each one - and we'll make the   //
+                  // line appear to be a singular-past-tense line about that individual key //
+                  ////////////////////////////////////////////////////////////////////////////
+                  processSummaryLine.setCount(1);
+                  processSummaryLine.pickMessage(true);
+
                   for(Serializable primaryKey : primaryKeys)
                   {
-                     HashMap<String, Serializable> map = toMap(processSummaryLine);
+                     HashMap<String, Serializable> map = toMap(processSummaryLine, false);
                      map.put("id", primaryKey);
                      apiOutput.add(map);
                   }
                }
                else
                {
-                  apiOutput.add(toMap(processSummaryLine));
+                  //////////////////////////////////////////////////////////////////////////
+                  // otherwise, handle a line without pkeys as a single output map/object //
+                  //////////////////////////////////////////////////////////////////////////
+                  HashMap<String, Serializable> map = toMap(processSummaryLine, true);
+                  apiOutput.add(map);
                }
             }
             else if(processSummaryLineInterface instanceof ProcessSummaryRecordLink processSummaryRecordLink)
@@ -219,12 +230,19 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static HashMap<String, Serializable> toMap(ProcessSummaryLine processSummaryLine)
+   private static HashMap<String, Serializable> toMap(ProcessSummaryLine processSummaryLine, boolean tryToIncludeCount)
    {
       HashMap<String, Serializable> map = initResultMapForProcessSummaryLine(processSummaryLine);
 
       String messagePrefix = getResultMapMessagePrefix(processSummaryLine);
-      map.put("message", messagePrefix + processSummaryLine.getMessage());
+
+      String messageSuffix = processSummaryLine.getMessage();
+      if(tryToIncludeCount && processSummaryLine.getCount() != null)
+      {
+         messageSuffix = processSummaryLine.getCount() + " " + messageSuffix;
+      }
+
+      map.put("message", messagePrefix + messageSuffix);
 
       return (map);
    }