CE-1328 Add job status messages

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStep.java

@@ -0,0 +1,287 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class GenericGarbageCollectorExecuteStep implements BackendStep
+{
+   private static final QLogger LOG = QLogger.getLogger(GenericGarbageCollectorExecuteStep.class);
+
+   private ProcessSummaryLine partitionsLine = new ProcessSummaryLine(Status.INFO)
+      .withSingularPastMessage("partition was processed")
+      .withPluralPastMessage("partitions were processed");
+
+   private ProcessSummaryLine deletedLine = new ProcessSummaryLine(Status.OK)
+      .withSingularPastMessage("record was deleted")
+      .withPluralPastMessage("records were deleted");
+
+   private ProcessSummaryLine warningLine = new ProcessSummaryLine(Status.WARNING, "had an warning");
+   private ProcessSummaryLine errorLine   = new ProcessSummaryLine(Status.ERROR, "had an error");
+
+   private AsyncJobCallback asyncJobCallback;
+
+   private Integer total        = null;
+   private Integer deletedSoFar = 0;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      String         tableName = runBackendStepInput.getValueString("table");
+      QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+      if(table == null)
+      {
+         throw new QUserFacingException("Unrecognized table: " + tableName);
+      }
+
+      String         fieldName = runBackendStepInput.getValueString("field");
+      QFieldMetaData field     = table.getFields().get(fieldName);
+      if(field == null)
+      {
+         throw new QUserFacingException("Unrecognized field: " + fieldName);
+      }
+
+      if(!QFieldType.DATE_TIME.equals(field.getType()) && !QFieldType.DATE.equals(field.getType()))
+      {
+         throw new QUserFacingException("Field " + field + " is not a date-time or date type field.");
+      }
+
+      Integer daysBack = runBackendStepInput.getValueInteger("daysBack");
+      if(daysBack == null || daysBack < 0)
+      {
+         throw new QUserFacingException("Illegal value for daysBack: " + daysBack + "; Must be positive.");
+      }
+
+      Integer maxPageSize = runBackendStepInput.getValueInteger("maxPageSize");
+      if(maxPageSize == null || maxPageSize < 0)
+      {
+         throw new QUserFacingException("Illegal value for maxPageSize: " + maxPageSize + "; Must be positive.");
+      }
+
+      asyncJobCallback = runBackendStepInput.getAsyncJobCallback();
+      long start = System.currentTimeMillis();
+      execute(table, field, daysBack, maxPageSize);
+      long end = System.currentTimeMillis();
+
+      deletedLine.prepareForFrontend(true);
+      partitionsLine.prepareForFrontend(true);
+
+      ArrayList<ProcessSummaryLineInterface> processSummary = new ArrayList<>();
+      processSummary.add(partitionsLine);
+      processSummary.add(deletedLine);
+      warningLine.addSelfToListIfAnyCount(processSummary);
+      errorLine.addSelfToListIfAnyCount(processSummary);
+      processSummary.add(new ProcessSummaryLine(Status.INFO, "Total time:  " + String.format("%,d", ((end - start) / 1000)) + " seconds"));
+      runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummary);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void execute(QTableMetaData table, QFieldMetaData field, Integer daysBack, Integer maxPageSize) throws QException
+   {
+      asyncJobCallback.updateStatus("Counting records");
+      Instant maxDate = Instant.now().minusSeconds(daysBack * 60 * 60 * 24);
+      Instant minDate = findMinDateInTable(table, field);
+
+      processDateRange(table, field, maxPageSize, minDate, maxDate);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void processDateRange(QTableMetaData table, QFieldMetaData field, Integer maxPageSize, Instant minDate, Instant maxDate) throws QException
+   {
+      partitionsLine.incrementCount();
+
+      LOG.info("Counting", logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      Integer count = count(table, field, minDate, maxDate);
+      LOG.info("Count", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+
+      if(this.total == null)
+      {
+         this.total = count;
+      }
+
+      if(count == 0)
+      {
+         LOG.info("0 rows in this partition - nothing to delete", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      }
+      else if(count <= maxPageSize)
+      {
+         asyncJobCallback.updateStatus("Deleting records", deletedSoFar, total);
+         LOG.info("Deleting", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+         delete(table, field, minDate, maxDate);
+         this.deletedSoFar += count;
+      }
+      else
+      {
+         if(deletedSoFar == 0)
+         {
+            asyncJobCallback.updateStatus("Partitioning table", deletedSoFar, total);
+         }
+
+         LOG.info("Too many rows", logPair("count", count), logPair("maxPageSize", maxPageSize), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+         partition(table, field, minDate, maxDate, count, maxPageSize);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void partition(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate, Integer count, Integer maxPageSize) throws QException
+   {
+      int  noOfPartitions    = (int) Math.ceil((float) count / (float) maxPageSize);
+      long milliDiff         = maxDate.toEpochMilli() - minDate.toEpochMilli();
+      long milliPerPartition = milliDiff / noOfPartitions;
+
+      if(milliPerPartition < 1000)
+      {
+         throw (new QUserFacingException("To find a maxPageSize under " + String.format("%,d", maxPageSize) + ", the partition size would become smaller than 1 second (between " + minDate + " and " + maxDate + " there are " + String.format("%,d", count) + " rows) - you must use a larger maxPageSize to continue."));
+      }
+
+      LOG.info("Partitioning", logPair("count", count), logPair("noOfPartitions", noOfPartitions), logPair("milliDiff", milliDiff), logPair("milliPerPartition", milliPerPartition), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      for(int i = 0; i < noOfPartitions; i++)
+      {
+         maxDate = minDate.plusMillis(milliPerPartition);
+         processDateRange(table, field, maxPageSize, minDate, maxDate);
+         minDate = maxDate;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void delete(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
+   {
+      DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(table.getName())
+         .withQueryFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate))));
+
+      deletedLine.incrementCount(deleteOutput.getDeletedRecordCount());
+
+      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
+      {
+         warningLine.incrementCount(deleteOutput.getRecordsWithWarnings().size());
+      }
+
+      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
+      {
+         errorLine.incrementCount(deleteOutput.getRecordsWithErrors().size());
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Integer count(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
+   {
+      Aggregate      countDateField = new Aggregate(field.getName(), AggregateOperator.COUNT).withFieldType(QFieldType.INTEGER);
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(table.getName());
+      aggregateInput.withFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate)));
+      aggregateInput.withAggregate(countDateField);
+
+      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+      List<AggregateResult> results         = aggregateOutput.getResults();
+      if(CollectionUtils.nullSafeIsEmpty(results))
+      {
+         throw (new QUserFacingException("Could not count rows table (null or empty aggregate result)."));
+      }
+
+      return (ValueUtils.getValueAsInteger(results.get(0).getAggregateValue(countDateField)));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant findMinDateInTable(QTableMetaData table, QFieldMetaData field) throws QException
+   {
+      Aggregate      minDate        = new Aggregate(field.getName(), AggregateOperator.MIN);
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(table.getName());
+      aggregateInput.withAggregate(minDate);
+
+      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+      List<AggregateResult> results         = aggregateOutput.getResults();
+      if(CollectionUtils.nullSafeIsEmpty(results))
+      {
+         throw (new QUserFacingException("Could not find min date value in table (null or empty aggregate result)."));
+      }
+
+      return (ValueUtils.getValueAsInstant(results.get(0).getAggregateValue(minDate)));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorProcessMetaDataProducer.java

@@ -0,0 +1,88 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** Generic process that can perform garbage collection on any table (at least,
+ ** any table with a date or date-time field).
+ **
+ ** When running, this process prompts for:
+ ** - table name
+ ** - field name (e.g., the date/date-time field on that table)
+ ** - daysBack - any records older than that many days ago will be deleted.
+ ** - maxPageSize - to avoid running "1 huge query", if there are more than
+ ** this number of records between the min-date in the table and the max-date
+ ** (based on daysBack), then the time range is partitioned recursively until
+ ** pages smaller than this parameter are found.  The partitioning attempts to
+ ** be smart (e.g., not just ÷ 2), by doing count / maxPageSize.
+ *******************************************************************************/
+public class GenericGarbageCollectorProcessMetaDataProducer extends MetaDataProducer<QProcessMetaData>
+{
+   public static final String NAME = "GenericGarbageCollector";
+
+
+
+   /*******************************************************************************
+    ** See class header for param descriptions.
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      QProcessMetaData processMetaData = new QProcessMetaData()
+         .withName(NAME)
+         .withIcon(new QIcon().withName("auto_delete"))
+         .withStepList(List.of(
+            new QFrontendStepMetaData()
+               .withName("input")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+               .withFormField(new QFieldMetaData("table", QFieldType.STRING))
+               .withFormField(new QFieldMetaData("field", QFieldType.STRING))
+               .withFormField(new QFieldMetaData("daysBack", QFieldType.INTEGER).withDefaultValue(90))
+               .withFormField(new QFieldMetaData("maxPageSize", QFieldType.INTEGER).withDefaultValue(100000)),
+            new QBackendStepMetaData()
+               .withName("execute")
+               .withCode(new QCodeReference(GenericGarbageCollectorExecuteStep.class)),
+            new QFrontendStepMetaData()
+               .withName("result")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS))
+         ));
+
+      return (processMetaData);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStepTest.java

@@ -0,0 +1,243 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for GenericGarbageCollectorExecuteStep 
+ *******************************************************************************/
+class GenericGarbageCollectorExecuteStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testErrors() throws Exception
+   {
+      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: null");
+
+      input.addValue("table", "notATable");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: notATable");
+
+      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: null");
+
+      input.addValue("field", "notAField");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: notAField");
+
+      input.addValue("field", "firstName");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("not a date");
+
+      input.addValue("field", "timestamp");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: null");
+
+      input.addValue("daysBack", "-1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: -1");
+
+      input.addValue("daysBack", "1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: null");
+
+      input.addValue("maxPageSize", "-1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: -1");
+
+      input.addValue("maxPageSize", "1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Could not find min date value in table");
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void runAndThrow(RunProcessInput input) throws Exception
+   {
+      input.setStartAfterStep(null);
+      input.setProcessUUID(null);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+      if(runProcessOutput.getException().isPresent())
+      {
+         throw (runProcessOutput.getException().get());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test30days() throws QException
+   {
+      insertAndRunGC(30);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(2, queryOutput.getRecords().size());
+      assertEquals(Set.of(4, 5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   @Disabled("memory aggregator is failing to return an aggregate when no rows found, which is throwing an error...")
+   void test100days() throws QException
+   {
+      insertAndRunGC(100);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(5, queryOutput.getRecords().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test10days() throws QException
+   {
+      insertAndRunGC(10);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(1, queryOutput.getRecords().size());
+      assertEquals(Set.of(5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test1day() throws QException
+   {
+      insertAndRunGC(1);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(0, queryOutput.getRecords().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test1dayPartitioned() throws QException
+   {
+      insertAndRunGC(1, 2);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(0, queryOutput.getRecords().size());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void insertAndRunGC(Integer daysBack) throws QException
+   {
+      insertAndRunGC(daysBack, 1000);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void insertAndRunGC(Integer daysBack, Integer maxPageSize) throws QException
+   {
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords()));
+      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
+      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      input.addValue("field", "timestamp");
+      input.addValue("daysBack", daysBack);
+      input.addValue("maxPageSize", maxPageSize);
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+      new RunProcessAction().execute(input);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QRecord> getPersonRecords()
+   {
+      List<QRecord> records = List.of(
+         new QRecord().withValue("id", 1).withValue("timestamp", Instant.now().minus(90, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 2).withValue("timestamp", Instant.now().minus(31, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 3).withValue("timestamp", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
+         new QRecord().withValue("id", 4).withValue("timestamp", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
+         new QRecord().withValue("id", 5).withValue("timestamp", Instant.now().minus(5, ChronoUnit.DAYS)));
+      return records;
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedviews/SavedViewProcessTests.java

@@ -192,6 +192,7 @@ class SavedViewProcessTests extends BaseTest
     **
     *******************************************************************************/
    @Test
+   @SuppressWarnings("unchecked")
    void testNotFoundThrowsProperly() throws QException
    {
       QInstance qInstance = QContext.getQInstance();
@@ -244,4 +245,4 @@ class SavedViewProcessTests extends BaseTest
       }
    }
 
-}
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogHeader.java

@@ -0,0 +1,316 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model;
+
+
+import java.time.Instant;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QAssociation;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for OutboundApiLog table
+ *******************************************************************************/
+public class OutboundAPILogHeader extends QRecordEntity
+{
+   public static final String TABLE_NAME = "outboundApiLogHeader";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField()
+   private Instant timestamp;
+
+   @QField(possibleValueSourceName = "outboundApiMethod")
+   private String method;
+
+   @QField(possibleValueSourceName = "outboundApiStatusCode")
+   private Integer statusCode;
+
+   @QField(label = "URL")
+   private String url;
+
+   @QAssociation(name = OutboundAPILogRequest.TABLE_NAME)
+   private List<OutboundAPILogRequest> outboundAPILogRequestList;
+
+   @QAssociation(name = OutboundAPILogResponse.TABLE_NAME)
+   private List<OutboundAPILogResponse> outboundAPILogResponseList;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    **
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for timestamp
+    **
+    *******************************************************************************/
+   public Instant getTimestamp()
+   {
+      return timestamp;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for timestamp
+    **
+    *******************************************************************************/
+   public void setTimestamp(Instant timestamp)
+   {
+      this.timestamp = timestamp;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for timestamp
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withTimestamp(Instant timestamp)
+   {
+      this.timestamp = timestamp;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for method
+    **
+    *******************************************************************************/
+   public String getMethod()
+   {
+      return method;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for method
+    **
+    *******************************************************************************/
+   public void setMethod(String method)
+   {
+      this.method = method;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for method
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withMethod(String method)
+   {
+      this.method = method;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for statusCode
+    **
+    *******************************************************************************/
+   public Integer getStatusCode()
+   {
+      return statusCode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for statusCode
+    **
+    *******************************************************************************/
+   public void setStatusCode(Integer statusCode)
+   {
+      this.statusCode = statusCode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for statusCode
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withStatusCode(Integer statusCode)
+   {
+      this.statusCode = statusCode;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for url
+    **
+    *******************************************************************************/
+   public String getUrl()
+   {
+      return url;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for url
+    **
+    *******************************************************************************/
+   public void setUrl(String url)
+   {
+      this.url = url;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for url
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withUrl(String url)
+   {
+      this.url = url;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundAPILogRequestList
+    *******************************************************************************/
+   public List<OutboundAPILogRequest> getOutboundAPILogRequestList()
+   {
+      return (this.outboundAPILogRequestList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundAPILogRequestList
+    *******************************************************************************/
+   public void setOutboundAPILogRequestList(List<OutboundAPILogRequest> outboundAPILogRequestList)
+   {
+      this.outboundAPILogRequestList = outboundAPILogRequestList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundAPILogRequestList
+    *******************************************************************************/
+   public OutboundAPILogHeader withOutboundAPILogRequestList(List<OutboundAPILogRequest> outboundAPILogRequestList)
+   {
+      this.outboundAPILogRequestList = outboundAPILogRequestList;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundAPILogResponseList
+    *******************************************************************************/
+   public List<OutboundAPILogResponse> getOutboundAPILogResponseList()
+   {
+      return (this.outboundAPILogResponseList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundAPILogResponseList
+    *******************************************************************************/
+   public void setOutboundAPILogResponseList(List<OutboundAPILogResponse> outboundAPILogResponseList)
+   {
+      this.outboundAPILogResponseList = outboundAPILogResponseList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundAPILogResponseList
+    *******************************************************************************/
+   public OutboundAPILogHeader withOutboundAPILogResponseList(List<OutboundAPILogResponse> outboundAPILogResponseList)
+   {
+      this.outboundAPILogResponseList = outboundAPILogResponseList;
+      return (this);
+   }
+
+}
+

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogMetaDataProvider.java

@@ -22,21 +22,33 @@
 package com.kingsrook.qqq.backend.module.api.model;
 
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogExtractStep;
+import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogLoadStep;
+import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogTransformStep;
 
 
 /*******************************************************************************
@@ -50,8 +62,15 @@ public class OutboundAPILogMetaDataProvider
     *******************************************************************************/
    public static void defineAll(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      definePossibleValueSources(qInstance);
+      definePossibleValueSources().forEach(pvs ->
-      defineOutboundAPILogTable(qInstance, backendName, backendDetailEnricher);
+      {
+         if(qInstance.getPossibleValueSource(pvs.getName()) == null)
+         {
+            qInstance.addPossibleValueSource(pvs);
+         }
+      });
+
+      qInstance.addTable(defineOutboundAPILogTable(backendName, backendDetailEnricher));
    }
 
 
@@ -59,9 +78,71 @@ public class OutboundAPILogMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void definePossibleValueSources(QInstance instance)
+   public static void defineNewVersion(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      instance.addPossibleValueSource(new QPossibleValueSource()
+      definePossibleValueSources().forEach(pvs ->
+      {
+         if(qInstance.getPossibleValueSource(pvs.getName()) == null)
+         {
+            qInstance.addPossibleValueSource(pvs);
+         }
+      });
+
+      qInstance.addTable(defineOutboundAPILogHeaderTable(backendName, backendDetailEnricher));
+      qInstance.addPossibleValueSource(defineOutboundAPILogHeaderPossibleValueSource());
+      qInstance.addTable(defineOutboundAPILogRequestTable(backendName, backendDetailEnricher));
+      qInstance.addTable(defineOutboundAPILogResponseTable(backendName, backendDetailEnricher));
+      defineJoins().forEach(join -> qInstance.add(join));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static void defineMigrationProcesses(QInstance qInstance, String sourceTableName)
+   {
+      qInstance.addProcess(StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName("migrateOutboundApiLogToHeaderChildProcess")
+         .withLabel("Migrate Outbound API Log Test to Header/Child")
+         .withIcon(new QIcon().withName("drive_file_move"))
+         .withTableName(sourceTableName)
+         .withSourceTable(sourceTableName)
+         .withDestinationTable(OutboundAPILogHeader.TABLE_NAME)
+         .withExtractStepClass(MigrateOutboundAPILogExtractStep.class)
+         .withTransformStepClass(MigrateOutboundAPILogTransformStep.class)
+         .withLoadStepClass(MigrateOutboundAPILogLoadStep.class)
+         .withReviewStepRecordFields(List.of(
+            new QFieldMetaData("url", QFieldType.INTEGER)
+         ))
+         .getProcessMetaData());
+
+      qInstance.addProcess(StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName("migrateOutboundApiLogToMongoDBProcess")
+         .withLabel("Migrate Outbound API Log Test to MongoDB")
+         .withIcon(new QIcon().withName("drive_file_move"))
+         .withTableName(sourceTableName)
+         .withSourceTable(sourceTableName)
+         .withDestinationTable(OutboundAPILog.TABLE_NAME + "MongoDB")
+         .withExtractStepClass(MigrateOutboundAPILogExtractStep.class)
+         .withTransformStepClass(MigrateOutboundAPILogTransformStep.class)
+         .withLoadStepClass(MigrateOutboundAPILogLoadStep.class)
+         .withReviewStepRecordFields(List.of(
+            new QFieldMetaData("url", QFieldType.INTEGER)
+         ))
+         .getProcessMetaData());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QPossibleValueSource> definePossibleValueSources()
+   {
+      List<QPossibleValueSource> rs = new ArrayList<>();
+
+      rs.add(new QPossibleValueSource()
          .withName("outboundApiMethod")
          .withType(QPossibleValueSourceType.ENUM)
          .withEnumValues(List.of(
@@ -72,7 +153,7 @@ public class OutboundAPILogMetaDataProvider
             new QPossibleValue<>("DELETE")
          )));
 
-      instance.addPossibleValueSource(new QPossibleValueSource()
+      rs.add(new QPossibleValueSource()
          .withName("outboundApiStatusCode")
          .withType(QPossibleValueSourceType.ENUM)
          .withEnumValues(List.of(
@@ -91,6 +172,8 @@ public class OutboundAPILogMetaDataProvider
             new QPossibleValue<>(503, "503 (Service Unavailable)"),
             new QPossibleValue<>(504, "500 (Gateway Timeout)")
          )));
+
+      return (rs);
    }
 
 
@@ -98,9 +181,8 @@ public class OutboundAPILogMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void defineOutboundAPILogTable(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   public static QTableMetaData defineOutboundAPILogTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-
       QTableMetaData tableMetaData = new QTableMetaData()
          .withName(OutboundAPILog.TABLE_NAME)
          .withLabel("Outbound API Log")
@@ -119,29 +201,8 @@ public class OutboundAPILogMetaDataProvider
       tableMetaData.getField("requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
       tableMetaData.getField("responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
 
-      tableMetaData.getField("method").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
+      addChipAdornmentToMethodField(tableMetaData);
-         .withValue(AdornmentType.ChipValues.colorValue("GET", AdornmentType.ChipValues.COLOR_INFO))
+      addChipAdornmentToStatusCodeField(tableMetaData);
-         .withValue(AdornmentType.ChipValues.colorValue("POST", AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue("DELETE", AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue("PATCH", AdornmentType.ChipValues.COLOR_WARNING))
-         .withValue(AdornmentType.ChipValues.colorValue("PUT", AdornmentType.ChipValues.COLOR_WARNING)));
-
-      tableMetaData.getField("statusCode").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
-         .withValue(AdornmentType.ChipValues.colorValue(200, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(201, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(204, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(207, AdornmentType.ChipValues.COLOR_INFO))
-         .withValue(AdornmentType.ChipValues.colorValue(400, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(401, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(403, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(404, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(422, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(429, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(500, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(502, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(503, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(504, AdornmentType.ChipValues.COLOR_ERROR))
-      );
 
       ///////////////////////////////////////////
       // these are the lengths of a MySQL TEXT //
@@ -160,6 +221,210 @@ public class OutboundAPILogMetaDataProvider
          backendDetailEnricher.accept(tableMetaData);
       }
 
-      qInstance.addTable(tableMetaData);
+      return (tableMetaData);
    }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void addChipAdornmentToStatusCodeField(QTableMetaData tableMetaData)
+   {
+      tableMetaData.getField("statusCode").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
+         .withValue(AdornmentType.ChipValues.colorValue(200, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(201, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(204, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(207, AdornmentType.ChipValues.COLOR_INFO))
+         .withValue(AdornmentType.ChipValues.colorValue(400, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(401, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(403, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(404, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(422, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(429, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(500, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(502, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(503, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(504, AdornmentType.ChipValues.COLOR_ERROR))
+      );
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void addChipAdornmentToMethodField(QTableMetaData tableMetaData)
+   {
+      tableMetaData.getField("method").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
+         .withValue(AdornmentType.ChipValues.colorValue("GET", AdornmentType.ChipValues.COLOR_INFO))
+         .withValue(AdornmentType.ChipValues.colorValue("POST", AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue("DELETE", AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue("PATCH", AdornmentType.ChipValues.COLOR_WARNING))
+         .withValue(AdornmentType.ChipValues.colorValue("PUT", AdornmentType.ChipValues.COLOR_WARNING)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineOutboundAPILogHeaderTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(OutboundAPILogHeader.TABLE_NAME)
+         .withLabel("Outbound API Log Header/Child")
+         .withIcon(new QIcon().withName("data_object"))
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(OutboundAPILogHeader.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
+         .withSection(new QFieldSection("request", new QIcon().withName("arrow_upward"), Tier.T2, List.of("method", "url", OutboundAPILogRequest.TABLE_NAME + ".requestBody")))
+         .withSection(new QFieldSection("response", new QIcon().withName("arrow_downward"), Tier.T2, List.of("statusCode", OutboundAPILogResponse.TABLE_NAME + ".responseBody")))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("timestamp")))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+
+      // tableMetaData.getField(OutboundAPILogRequest.TABLE_NAME + ".requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+      // tableMetaData.getField(OutboundAPILogResponse.TABLE_NAME + ".responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+
+      addChipAdornmentToMethodField(tableMetaData);
+      addChipAdornmentToStatusCodeField(tableMetaData);
+
+      tableMetaData.withAssociation(new Association()
+         .withName(OutboundAPILogRequest.TABLE_NAME)
+         .withAssociatedTableName(OutboundAPILogRequest.TABLE_NAME)
+         .withJoinName(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogRequest.TABLE_NAME)));
+
+      tableMetaData.withAssociation(new Association()
+         .withName(OutboundAPILogResponse.TABLE_NAME)
+         .withAssociatedTableName(OutboundAPILogResponse.TABLE_NAME)
+         .withJoinName(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogResponse.TABLE_NAME)));
+
+      tableMetaData.withExposedJoin(new ExposedJoin()
+         .withJoinTable(OutboundAPILogRequest.TABLE_NAME)
+         .withJoinPath(List.of(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogRequest.TABLE_NAME))));
+
+      tableMetaData.withExposedJoin(new ExposedJoin()
+         .withJoinTable(OutboundAPILogResponse.TABLE_NAME)
+         .withJoinPath(List.of(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogResponse.TABLE_NAME))));
+
+      tableMetaData.getField("url").withMaxLength(4096).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+      tableMetaData.getField("url").withFieldAdornment(AdornmentType.Size.XLARGE.toAdornment());
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(tableMetaData);
+      }
+
+      return (tableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineOutboundAPILogRequestTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(OutboundAPILogRequest.TABLE_NAME)
+         .withLabel("Outbound API Log Request")
+         .withIcon(new QIcon().withName("arrow_upward"))
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(OutboundAPILogRequest.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "outboundApiLogHeaderId")))
+         .withSection(new QFieldSection("request", new QIcon().withName("arrow_upward"), Tier.T2, List.of("requestBody")))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+
+      tableMetaData.getField("requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+
+      //////////////////////////////////////////////
+      // this is the length of a MySQL MEDIUMTEXT //
+      //////////////////////////////////////////////
+      tableMetaData.getField("requestBody").withMaxLength(16_777_215).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(tableMetaData);
+      }
+
+      return (tableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineOutboundAPILogResponseTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(OutboundAPILogResponse.TABLE_NAME)
+         .withLabel("Outbound API Log Response")
+         .withIcon(new QIcon().withName("arrow_upward"))
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(OutboundAPILogResponse.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "outboundApiLogHeaderId")))
+         .withSection(new QFieldSection("response", new QIcon().withName("arrow_upward"), Tier.T2, List.of("responseBody")))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+
+      tableMetaData.getField("responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+
+      //////////////////////////////////////////////
+      // this is the length of a MySQL MEDIUMTEXT //
+      //////////////////////////////////////////////
+      tableMetaData.getField("responseBody").withMaxLength(16_777_215).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(tableMetaData);
+      }
+
+      return (tableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QJoinMetaData> defineJoins()
+   {
+      List<QJoinMetaData> rs = new ArrayList<>();
+
+      rs.add(new QJoinMetaData()
+         .withLeftTable(OutboundAPILogHeader.TABLE_NAME)
+         .withRightTable(OutboundAPILogRequest.TABLE_NAME)
+         .withInferredName()
+         .withType(JoinType.ONE_TO_ONE)
+         .withJoinOn(new JoinOn("id", "outboundApiLogHeaderId")));
+
+      rs.add(new QJoinMetaData()
+         .withLeftTable(OutboundAPILogHeader.TABLE_NAME)
+         .withRightTable(OutboundAPILogResponse.TABLE_NAME)
+         .withInferredName()
+         .withType(JoinType.ONE_TO_ONE)
+         .withJoinOn(new JoinOn("id", "outboundApiLogHeaderId")));
+
+      return (rs);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static QPossibleValueSource defineOutboundAPILogHeaderPossibleValueSource()
+   {
+      return QPossibleValueSource.newForTable(OutboundAPILogHeader.TABLE_NAME);
+   }
+
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogRequest.java

@@ -0,0 +1,165 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for OutboundApiLogRequest table
+ *******************************************************************************/
+public class OutboundAPILogRequest extends QRecordEntity
+{
+   public static final String TABLE_NAME = "outboundApiLogRequest";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(possibleValueSourceName = OutboundAPILogHeader.TABLE_NAME)
+   private Integer outboundApiLogHeaderId;
+
+   @QField()
+   private String requestBody;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogRequest()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogRequest(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    **
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    **
+    *******************************************************************************/
+   public OutboundAPILogRequest withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for requestBody
+    *******************************************************************************/
+   public String getRequestBody()
+   {
+      return (this.requestBody);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for requestBody
+    *******************************************************************************/
+   public void setRequestBody(String requestBody)
+   {
+      this.requestBody = requestBody;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for requestBody
+    *******************************************************************************/
+   public OutboundAPILogRequest withRequestBody(String requestBody)
+   {
+      this.requestBody = requestBody;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public Integer getOutboundApiLogHeaderId()
+   {
+      return (this.outboundApiLogHeaderId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public void setOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public OutboundAPILogRequest withOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+      return (this);
+   }
+
+}
+

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogResponse.java

@@ -0,0 +1,165 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for OutboundApiLogResponse table
+ *******************************************************************************/
+public class OutboundAPILogResponse extends QRecordEntity
+{
+   public static final String TABLE_NAME = "outboundApiLogResponse";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(possibleValueSourceName = OutboundAPILogHeader.TABLE_NAME)
+   private Integer outboundApiLogHeaderId;
+
+   @QField()
+   private String responseBody;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogResponse()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogResponse(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    **
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    **
+    *******************************************************************************/
+   public OutboundAPILogResponse withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for responseBody
+    *******************************************************************************/
+   public String getResponseBody()
+   {
+      return (this.responseBody);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for responseBody
+    *******************************************************************************/
+   public void setResponseBody(String responseBody)
+   {
+      this.responseBody = responseBody;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for responseBody
+    *******************************************************************************/
+   public OutboundAPILogResponse withResponseBody(String responseBody)
+   {
+      this.responseBody = responseBody;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public Integer getOutboundApiLogHeaderId()
+   {
+      return (this.outboundApiLogHeaderId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public void setOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public OutboundAPILogResponse withOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+      return (this);
+   }
+
+}
+

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogExtractStep.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MigrateOutboundAPILogExtractStep extends ExtractViaQueryStep
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected void customizeInputPreQuery(QueryInput queryInput)
+   {
+      queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
+   }
+
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogLoadStep.java

@@ -0,0 +1,54 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
+
+
+/*******************************************************************************
+ ** store outboundApiLogHeaders and maybe more...
+ *******************************************************************************/
+public class MigrateOutboundAPILogLoadStep extends LoadViaInsertStep
+{
+   private static final QLogger LOG = QLogger.getLogger(MigrateOutboundAPILogLoadStep.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      super.runOnePage(runBackendStepInput, runBackendStepOutput);
+
+      ///////////////////////////////////////
+      // todo - track what we've migrated? //
+      ///////////////////////////////////////
+   }
+
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogTransformStep.java

@@ -0,0 +1,127 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogHeader;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogRequest;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogResponse;
+
+
+/*******************************************************************************
+ ** migrate records from original (singular) outboundApiLog table to new split-up
+ ** version (outboundApiLogHeader)
+ *******************************************************************************/
+public class MigrateOutboundAPILogTransformStep extends AbstractTransformStep
+{
+   private static final QLogger LOG = QLogger.getLogger(MigrateOutboundAPILogTransformStep.class);
+
+   private ProcessSummaryLine okToInsertLine = StandardProcessSummaryLineProducer.getOkToInsertLine();
+   private ProcessSummaryLine errorLine      = StandardProcessSummaryLineProducer.getErrorLine();
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   /*
+   @Override
+   public Integer getOverrideRecordPipeCapacity(RunBackendStepInput runBackendStepInput)
+   {
+      return (100);
+   }
+   */
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
+   {
+      ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
+      okToInsertLine.addSelfToListIfAnyCount(rs);
+      errorLine.addSelfToListIfAnyCount(rs);
+      return (rs);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      runBackendStepOutput.addValue("counter", 0);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      int counter = runBackendStepOutput.getValueInteger("counter") + runBackendStepInput.getRecords().size();
+      runBackendStepOutput.addValue("counter", counter);
+      runBackendStepInput.getAsyncJobCallback().updateStatus("Migrating records (at #" + String.format("%,d", counter) + ")");
+
+      String destinationTable = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE);
+
+      for(QRecord record : runBackendStepInput.getRecords())
+      {
+         okToInsertLine.incrementCountAndAddPrimaryKey(record.getValue("id"));
+
+         if(destinationTable.equals(OutboundAPILogHeader.TABLE_NAME))
+         {
+            OutboundAPILogHeader outboundAPILogHeader = new OutboundAPILogHeader(record);
+            outboundAPILogHeader.withOutboundAPILogRequestList(List.of(new OutboundAPILogRequest().withRequestBody(record.getValueString("requestBody"))));
+            outboundAPILogHeader.withOutboundAPILogResponseList(List.of(new OutboundAPILogResponse().withResponseBody(record.getValueString("responseBody"))));
+            runBackendStepOutput.addRecord(outboundAPILogHeader.toQRecord());
+         }
+         else
+         {
+            ///////////////////////////////////////////////////////////////////
+            // for the mongodb migration, just pass records straight through //
+            ///////////////////////////////////////////////////////////////////
+            record.setValue("id", null);
+            runBackendStepOutput.addRecord(record);
+         }
+      }
+   }
+
+}

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/BaseTest.java

@@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 
@@ -46,6 +47,7 @@ public class BaseTest
    void baseBeforeEach()
    {
       QContext.init(TestUtils.defineInstance(), new QSession());
+      MemoryRecordStore.getInstance().reset();
    }
 
 
@@ -57,6 +59,7 @@ public class BaseTest
    void baseAfterEach()
    {
       QContext.clear();
+      MemoryRecordStore.getInstance().reset();
    }
 
 

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogProcessTest.java

@@ -0,0 +1,98 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.module.api.BaseTest;
+import com.kingsrook.qqq.backend.module.api.TestUtils;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogHeader;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogMetaDataProvider;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for MigrateOutboundAPILog process
+ *******************************************************************************/
+class MigrateOutboundAPILogProcessTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void beforeEach() throws QException
+   {
+      MemoryRecordStore.getInstance().reset();
+      OutboundAPILogMetaDataProvider.defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
+      OutboundAPILogMetaDataProvider.defineNewVersion(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
+      OutboundAPILogMetaDataProvider.defineMigrationProcesses(QContext.getQInstance(), OutboundAPILog.TABLE_NAME);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      new InsertAction().execute(new InsertInput(OutboundAPILog.TABLE_NAME).withRecordEntity(new OutboundAPILog()
+         .withMethod("POST")
+         .withUrl("www.google.com")
+         .withRequestBody("please")
+         .withResponseBody("you're welcome")
+         .withStatusCode(201)
+      ));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName("migrateOutboundApiLogToHeaderChildProcess");
+      input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter()));
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+
+      List<OutboundAPILogHeader> outboundApiLogHeaderList = new QueryAction().execute(new QueryInput(OutboundAPILogHeader.TABLE_NAME).withIncludeAssociations(true)).getRecordEntities(OutboundAPILogHeader.class);
+      assertEquals(1, outboundApiLogHeaderList.size());
+      assertEquals("POST", outboundApiLogHeaderList.get(0).getMethod());
+      assertEquals(201, outboundApiLogHeaderList.get(0).getStatusCode());
+      assertEquals(1, outboundApiLogHeaderList.get(0).getOutboundAPILogRequestList().size());
+      assertEquals("please", outboundApiLogHeaderList.get(0).getOutboundAPILogRequestList().get(0).getRequestBody());
+      assertEquals(1, outboundApiLogHeaderList.get(0).getOutboundAPILogResponseList().size());
+      assertEquals("you're welcome", outboundApiLogHeaderList.get(0).getOutboundAPILogResponseList().get(0).getResponseBody());
+   }
+
+}

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInputFieldsContainer.java

@@ -43,6 +43,32 @@ public class ApiProcessInputFieldsContainer
 
 
 
+   /***************************************************************************
+    ** factory method to build one of these containers using all of the input fields
+    ** in a process
+    ***************************************************************************/
+   public static ApiProcessInputFieldsContainer forAllInputFields(QProcessMetaData process)
+   {
+      return forFields(process.getInputFields());
+   }
+
+
+
+   /***************************************************************************
+    ** factory method to build one of these containers using a list of fields.
+    ***************************************************************************/
+   public static ApiProcessInputFieldsContainer forFields(List<QFieldMetaData> fields)
+   {
+      ApiProcessInputFieldsContainer container = new ApiProcessInputFieldsContainer();
+      for(QFieldMetaData inputField : CollectionUtils.nonNullList(fields))
+      {
+         container.withField(inputField);
+      }
+      return (container);
+   }
+
+
+
    /*******************************************************************************
     ** find all input fields in frontend steps of the process, and add them as fields
     ** in this container.

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessSummaryListOutput.java

@@ -48,7 +48,9 @@ import org.eclipse.jetty.http.HttpStatus;
 
 
 /*******************************************************************************
- **
+ ** For a process that puts "processResults" in its output (as a list of
+ ** ProcessSummaryLineInterface objects) - this class converts such an object
+ ** to a suitable ApiProcessOutput.
  *******************************************************************************/
 public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
 {
@@ -143,22 +145,31 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
          {
             if(processSummaryLineInterface instanceof ProcessSummaryLine processSummaryLine)
             {
-               processSummaryLine.setCount(1);
-               processSummaryLine.pickMessage(true);
-
                List<Serializable> primaryKeys = processSummaryLine.getPrimaryKeys();
                if(CollectionUtils.nullSafeHasContents(primaryKeys))
                {
+                  ////////////////////////////////////////////////////////////////////////////
+                  // if there are primary keys in the line, then we'll loop over those, and //
+                  // output an object in the API output for each one - and we'll make the   //
+                  // line appear to be a singular-past-tense line about that individual key //
+                  ////////////////////////////////////////////////////////////////////////////
+                  processSummaryLine.setCount(1);
+                  processSummaryLine.pickMessage(true);
+
                   for(Serializable primaryKey : primaryKeys)
                   {
-                     HashMap<String, Serializable> map = toMap(processSummaryLine);
+                     HashMap<String, Serializable> map = toMap(processSummaryLine, false);
                      map.put("id", primaryKey);
                      apiOutput.add(map);
                   }
                }
                else
                {
-                  apiOutput.add(toMap(processSummaryLine));
+                  //////////////////////////////////////////////////////////////////////////
+                  // otherwise, handle a line without pkeys as a single output map/object //
+                  //////////////////////////////////////////////////////////////////////////
+                  HashMap<String, Serializable> map = toMap(processSummaryLine, true);
+                  apiOutput.add(map);
                }
             }
             else if(processSummaryLineInterface instanceof ProcessSummaryRecordLink processSummaryRecordLink)
@@ -219,12 +230,19 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static HashMap<String, Serializable> toMap(ProcessSummaryLine processSummaryLine)
+   private static HashMap<String, Serializable> toMap(ProcessSummaryLine processSummaryLine, boolean tryToIncludeCount)
    {
       HashMap<String, Serializable> map = initResultMapForProcessSummaryLine(processSummaryLine);
 
       String messagePrefix = getResultMapMessagePrefix(processSummaryLine);
-      map.put("message", messagePrefix + processSummaryLine.getMessage());
+
+      String messageSuffix = processSummaryLine.getMessage();
+      if(tryToIncludeCount && processSummaryLine.getCount() != null)
+      {
+         messageSuffix = processSummaryLine.getCount() + " " + messageSuffix;
+      }
+
+      map.put("message", messagePrefix + messageSuffix);
 
       return (map);
    }