Add LoadViaInsertOrUpdateStep; make PVS field labels not have Id suffix; add populateFromQRecord

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -79,6 +79,11 @@ public class QInstanceEnricher
 
    private final QInstance qInstance;
 
+   //////////////////////////////////////////////////////////
+   // todo - come up w/ a way for app devs to set configs! //
+   //////////////////////////////////////////////////////////
+   private boolean configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels = true;
+
 
 
    /*******************************************************************************
@@ -229,7 +234,14 @@ public class QInstanceEnricher
    {
       if(!StringUtils.hasContent(field.getLabel()))
       {
-         field.setLabel(nameToLabel(field.getName()));
+         if(configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels && StringUtils.hasContent(field.getPossibleValueSourceName()) && field.getName() != null && field.getName().endsWith("Id"))
+         {
+            field.setLabel(nameToLabel(field.getName().substring(0, field.getName().length() - 2)));
+         }
+         else
+         {
+            field.setLabel(nameToLabel(field.getName()));
+         }
       }
 
       //////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java

@@ -60,16 +60,32 @@ public abstract class QRecordEntity
       try
       {
          T entity = c.getConstructor().newInstance();
+         entity.populateFromQRecord(qRecord);
+         return (entity);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error building entity from qRecord.", e));
+      }
+   }
 
-         List<QRecordEntityField> fieldList = getFieldList(c);
+
+
+   /*******************************************************************************
+    ** Build an entity of this QRecord type from a QRecord
+    **
+    *******************************************************************************/
+   protected <T extends QRecordEntity> void populateFromQRecord(QRecord qRecord) throws QException
+   {
+      try
+      {
+         List<QRecordEntityField> fieldList = getFieldList(this.getClass());
          for(QRecordEntityField qRecordEntityField : fieldList)
          {
             Serializable value      = qRecord.getValue(qRecordEntityField.getFieldName());
             Object       typedValue = qRecordEntityField.convertValueType(value);
-            qRecordEntityField.getSetter().invoke(entity, typedValue);
+            qRecordEntityField.getSetter().invoke(this, typedValue);
          }
-
-         return (entity);
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java

@@ -0,0 +1,113 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Generic implementation of a LoadStep - that runs Insert and/or Update
+ ** actions for the destination table - where the presence or absence of the
+ ** record's primaryKey field is the indicator for which to do.  e.g., it assumes
+ ** auto-generated ids, to be populated upon insert.
+ *******************************************************************************/
+public class LoadViaInsertOrUpdateStep extends AbstractLoadStep
+{
+   public static final String FIELD_DESTINATION_TABLE = "destinationTable";
+
+
+
+   /*******************************************************************************
+    ** Execute the backend step - using the request as input, and the result as output.
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      QTableMetaData tableMetaData   = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
+      List<QRecord>  recordsToInsert = new ArrayList<>();
+      List<QRecord>  recordsToUpdate = new ArrayList<>();
+      for(QRecord record : runBackendStepInput.getRecords())
+      {
+         if(record.getValue(tableMetaData.getPrimaryKeyField()) == null)
+         {
+            recordsToInsert.add(record);
+         }
+         else
+         {
+            recordsToUpdate.add(record);
+         }
+      }
+
+      if(!recordsToInsert.isEmpty())
+      {
+         InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance());
+         insertInput.setSession(runBackendStepInput.getSession());
+         insertInput.setTableName(tableMetaData.getName());
+         insertInput.setRecords(runBackendStepInput.getRecords());
+         getTransaction().ifPresent(insertInput::setTransaction);
+         InsertOutput insertOutput = new InsertAction().execute(insertInput);
+         runBackendStepOutput.getRecords().addAll(insertOutput.getRecords());
+      }
+
+      if(!recordsToUpdate.isEmpty())
+      {
+         UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance());
+         updateInput.setSession(runBackendStepInput.getSession());
+         updateInput.setTableName(tableMetaData.getName());
+         updateInput.setRecords(runBackendStepInput.getRecords());
+         getTransaction().ifPresent(updateInput::setTransaction);
+         UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
+         runBackendStepOutput.getRecords().addAll(updateOutput.getRecords());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
+   {
+      InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance());
+      insertInput.setSession(runBackendStepInput.getSession());
+      insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
+
+      return (Optional.of(new InsertAction().openTransaction(insertInput)));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java

@@ -80,10 +80,11 @@ public class StreamedETLWithFrontendProcess
    public static final String FIELD_VALIDATION_SUMMARY       = "validationSummary"; // List<ProcessSummaryLine>
    public static final String FIELD_PROCESS_SUMMARY          = "processResults"; // List<ProcessSummaryLine>
 
-   public static final String DEFAULT_PREVIEW_MESSAGE_FOR_INSERT = "This is a preview of the records that will be created.";
+   public static final String DEFAULT_PREVIEW_MESSAGE_FOR_INSERT           = "This is a preview of the records that will be created.";
-   public static final String DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE = "This is a preview of the records that will be updated.";
+   public static final String DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE           = "This is a preview of the records that will be updated.";
-   public static final String DEFAULT_PREVIEW_MESSAGE_FOR_DELETE = "This is a preview of the records that will be deleted.";
+   public static final String DEFAULT_PREVIEW_MESSAGE_FOR_INSERT_OR_UPDATE = "This is a preview of the records that will be inserted or updated.";
-   public static final String FIELD_PREVIEW_MESSAGE              = "previewMessage";
+   public static final String DEFAULT_PREVIEW_MESSAGE_FOR_DELETE           = "This is a preview of the records that will be deleted.";
+   public static final String FIELD_PREVIEW_MESSAGE                        = "previewMessage";
 
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java

@@ -245,32 +245,32 @@ public class GenerateReportActionTest
       Map<String, String>           row      = iterator.next();
       assertEquals(6, list.size());
 
-      assertThat(row.get("Home State Id")).isEqualTo("IL");
+      assertThat(row.get("Home State")).isEqualTo("IL");
       assertThat(row.get("Last Name")).isEqualTo("Jonson");
       assertThat(row.get("Quantity")).isNull();
 
       row = iterator.next();
-      assertThat(row.get("Home State Id")).isEqualTo("IL");
+      assertThat(row.get("Home State")).isEqualTo("IL");
       assertThat(row.get("Last Name")).isEqualTo("Jones");
       assertThat(row.get("Quantity")).isEqualTo("3");
 
       row = iterator.next();
-      assertThat(row.get("Home State Id")).isEqualTo("IL");
+      assertThat(row.get("Home State")).isEqualTo("IL");
       assertThat(row.get("Last Name")).isEqualTo("Kelly");
       assertThat(row.get("Quantity")).isEqualTo("4");
 
       row = iterator.next();
-      assertThat(row.get("Home State Id")).isEqualTo("IL");
+      assertThat(row.get("Home State")).isEqualTo("IL");
       assertThat(row.get("Last Name")).isEqualTo("Keller");
       assertThat(row.get("Quantity")).isEqualTo("5");
 
       row = iterator.next();
-      assertThat(row.get("Home State Id")).isEqualTo("IL");
+      assertThat(row.get("Home State")).isEqualTo("IL");
       assertThat(row.get("Last Name")).isEqualTo("Kelkhoff");
       assertThat(row.get("Quantity")).isEqualTo("6");
 
       row = iterator.next();
-      assertThat(row.get("Home State Id")).isEqualTo("MO");
+      assertThat(row.get("Home State")).isEqualTo("MO");
       assertThat(row.get("Last Name")).isEqualTo("Kelkhoff");
       assertThat(row.get("Quantity")).isEqualTo("7");
    }
@@ -299,12 +299,12 @@ public class GenerateReportActionTest
       Iterator<Map<String, String>> iterator = list.iterator();
       Map<String, String>           row      = iterator.next();
       assertEquals(2, list.size());
-      assertThat(row.get("Home State Id")).isEqualTo("MO");
+      assertThat(row.get("Home State")).isEqualTo("MO");
       assertThat(row.get("Last Name")).isNull();
       assertThat(row.get("Quantity")).isEqualTo("7");
 
       row = iterator.next();
-      assertThat(row.get("Home State Id")).isEqualTo("IL");
+      assertThat(row.get("Home State")).isEqualTo("IL");
       assertThat(row.get("Last Name")).isNull();
       assertThat(row.get("Quantity")).isEqualTo("18");
    }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java

@@ -108,6 +108,23 @@ class QInstanceEnricherTest
 
 
 
+   /*******************************************************************************
+    ** Test that a field missing a label gets the default label applied (name w/ UC-first)
+    ** w/ Id stripped from the end, because it's a PVS
+    **
+    *******************************************************************************/
+   @Test
+   public void test_nullFieldLabelComesFromNameWithoutIdForPossibleValues()
+   {
+      QInstance      qInstance        = TestUtils.defineInstance();
+      QFieldMetaData homeStateIdField = qInstance.getTable("person").getField("homeStateId");
+      assertNull(homeStateIdField.getLabel());
+      new QInstanceEnricher(qInstance).enrich();
+      assertEquals("Home State", homeStateIdField.getLabel());
+   }
+
+
+
    /*******************************************************************************
     ** Test that a fieldSection missing a label gets the default label applied (name w/ UC-first).
     **

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java

@@ -111,6 +111,45 @@ public class StreamedETLWithFrontendProcessTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testLoadViaInsertOrUpdate() throws QException
+   {
+      QInstance instance = TestUtils.defineInstance();
+
+      /////////////////////////////////////////////////////////////////////////////////
+      // define the process - an ELT from Shapes to Shapes - inserting 1, updating 2 //
+      /////////////////////////////////////////////////////////////////////////////////
+      QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
+         TestUtils.TABLE_NAME_SHAPE,
+         TestUtils.TABLE_NAME_SHAPE,
+         ExtractViaQueryStep.class,
+         TestTransformShapeToMaybeNewShape.class,
+         LoadViaInsertOrUpdateStep.class);
+      process.setName("test");
+      process.setTableName(TestUtils.TABLE_NAME_SHAPE);
+      instance.addProcess(process);
+
+      TestUtils.insertDefaultShapes(instance);
+
+      /////////////////////
+      // run the process //
+      /////////////////////
+      runProcess(instance, process);
+
+      List<QRecord> postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_SHAPE);
+      assertEquals(4, postList.size());
+      assertThat(postList)
+         .as("Should have inserted a new Square").anyMatch(qr -> qr.getValue("name").equals("a new Square"))
+         .as("Should have left old Square alone").anyMatch(qr -> qr.getValue("name").equals("Square"))
+         .as("Should have updated Triangle").anyMatch(qr -> qr.getValue("name").equals("an updated Triangle"))
+         .as("Should have updated Circle").anyMatch(qr -> qr.getValue("name").equals("an updated Circle"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -368,6 +407,49 @@ public class StreamedETLWithFrontendProcessTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class TestTransformShapeToMaybeNewShape extends AbstractTransformStep
+   {
+
+      /*******************************************************************************
+       ** Execute the backend step - using the request as input, and the result as output.
+       **
+       *******************************************************************************/
+      @Override
+      public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+      {
+         for(QRecord qRecord : runBackendStepInput.getRecords())
+         {
+            String name = qRecord.getValueString("name");
+            if(name.equals("Square"))
+            {
+               QRecord toInsertRecord = new QRecord();
+               toInsertRecord.setValue("name", "a new Square");
+               runBackendStepOutput.getRecords().add(toInsertRecord);
+            }
+            else
+            {
+               QRecord toUpdateRecord = new QRecord();
+               toUpdateRecord.setValue("id", qRecord.getValueInteger("id"));
+               toUpdateRecord.setValue("name", "an updated " + name);
+               runBackendStepOutput.getRecords().add(toUpdateRecord);
+            }
+         }
+      }
+
+
+
+      @Override
+      public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
+      {
+         return null;
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/