Merged dev into feature/CE-798-quick-filters

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -31,6 +31,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
@@ -1251,7 +1252,25 @@ public class QInstanceValidator
                         {
                            if(fieldMetaData.getDefaultValue() != null && fieldMetaData.getDefaultValue() instanceof QCodeReference codeReference)
                            {
-                              validateSimpleCodeReference("Process " + processName + " backend step code reference: ", codeReference, BackendStep.class);
+                              ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                              // by default, assume that any process field which is a QCodeReference should be a reference to a BackendStep... //
+                              // but... allow a secondary field name to be set, to tell us what class to *actually* expect here...             //
+                              ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                              Class<?> expectedClass = BackendStep.class;
+                              try
+                              {
+                                 Optional<QFieldMetaData> expectedTypeField = backendStepMetaData.getInputMetaData().getField(fieldMetaData.getName() + "_expectedType");
+                                 if(expectedTypeField.isPresent() && expectedTypeField.get().getDefaultValue() != null)
+                                 {
+                                    expectedClass = Class.forName(ValueUtils.getValueAsString(expectedTypeField.get().getDefaultValue()));
+                                 }
+                              }
+                              catch(Exception e)
+                              {
+                                 warn("Error loading expectedType for field [" + fieldMetaData.getName() + "] in process [" + processName + "]: " + e.getMessage());
+                              }
+
+                              validateSimpleCodeReference("Process " + processName + " code reference: ", codeReference, expectedClass);
                            }
                         }
                      }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -746,12 +746,22 @@ public class QInstance
 
 
    /*******************************************************************************
-    ** Setter for hasBeenValidated
+    ** If pass a QInstanceValidationKey (which can only be instantiated by the validator),
+    ** then the hasBeenValidated field will be set to true.
     **
+    ** Else, if passed a null, hasBeenValidated will be reset to false - e.g., to
+    ** re-trigger validation (can be useful in tests).
     *******************************************************************************/
    public void setHasBeenValidated(QInstanceValidationKey key)
    {
-      this.hasBeenValidated = true;
+      if(key == null)
+      {
+         this.hasBeenValidated = false;
+      }
+      else
+      {
+         this.hasBeenValidated = true;
+      }
    }
 
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java

@@ -141,28 +141,38 @@ public class FilesystemImporterMetaDataTemplate
 
 
    /*******************************************************************************
-    **
+    ** Set up importRecord table being built by this template to hve an automation-
+    ** status field on it, and an automation details object attached to it.
     *******************************************************************************/
-   public void addAutomationStatusField(QTableMetaData table, QFieldMetaData automationStatusField)
+   public void addImportRecordAutomations(QFieldMetaData automationStatusField, QTableAutomationDetails automationDetails)
    {
-      table.addField(automationStatusField);
+      getImportRecordTable().addField(automationStatusField);
-      table.getSections().get(1).getFieldNames().add(0, automationStatusField.getName());
+      getImportRecordTable().getSections().get(1).getFieldNames().add(0, automationStatusField.getName());
+      getImportRecordTable().withAutomationDetails(automationDetails);
    }
 
 
 
    /*******************************************************************************
+    ** Add 1 process as a post-insert automation-action on this template's importRecord
+    ** table.
     **
+    ** The automation action is returned - which you may want for changing things, e.g.,
+    ** its priority (e.g., addImportRecordPostInsertAutomationAction(...).withPriority(1);
     *******************************************************************************/
-   public TableAutomationAction addStandardPostInsertAutomation(QTableMetaData table, QTableAutomationDetails automationDetails, String processName)
+   public TableAutomationAction addImportRecordPostInsertAutomationAction(String processName)
    {
+      if(getImportRecordTable().getAutomationDetails() == null)
+      {
+         throw (new IllegalStateException(getImportRecordTable().getName() + " does not have automationDetails - do you need to call addAutomations first?"));
+      }
+
       TableAutomationAction action = new TableAutomationAction()
-         .withName(table.getName() + "PostInsert")
+         .withName(processName)
          .withTriggerEvent(TriggerEvent.POST_INSERT)
          .withProcessName(processName);
 
-      table.withAutomationDetails(automationDetails
+      getImportRecordTable().getAutomationDetails().withAction(action);
-         .withAction(action));
 
       return (action);
    }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java

@@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
 
 
 import java.io.Serializable;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@@ -62,6 +64,15 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
                .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_PATH, QFieldType.STRING))
                .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, QFieldType.STRING))
                .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, QFieldType.STRING))
+
+               //////////////////////////////////////////////////////////////////////////////////////
+               // define a QCodeReference - expected to be of type Function<QRecord, Serializable> //
+               // make sure the QInstanceValidator knows that the QCodeReference should be a       //
+               // Function (not a BackendStep, which is the default for process fields)            //
+               //////////////////////////////////////////////////////////////////////////////////////
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_VALUE_SUPPLIER, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_VALUE_SUPPLIER + "_expectedType", QFieldType.STRING)
+                  .withDefaultValue(Function.class.getName()))
             )));
    }
 
@@ -186,4 +197,15 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withImportSecurityValueSupplierFunction(Class<? extends Function<QRecord, Serializable>> supplierFunction)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_VALUE_SUPPLIER, new QCodeReference(supplierFunction));
+      return (this);
+   }
+
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java

@@ -33,7 +33,9 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
 import java.util.UUID;
+import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -41,6 +43,7 @@ import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
 import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -53,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -83,8 +87,9 @@ public class FilesystemImporterStep implements BackendStep
    public static final String FIELD_IMPORT_FILE_TABLE   = "importFileTable";
    public static final String FIELD_IMPORT_RECORD_TABLE = "importRecordTable";
 
-   public static final String FIELD_IMPORT_SECURITY_FIELD_NAME  = "importSecurityFieldName";
+   public static final String FIELD_IMPORT_SECURITY_FIELD_NAME     = "importSecurityFieldName";
-   public static final String FIELD_IMPORT_SECURITY_FIELD_VALUE = "importSecurityFieldValue";
+   public static final String FIELD_IMPORT_SECURITY_FIELD_VALUE    = "importSecurityFieldValue";
+   public static final String FIELD_IMPORT_SECURITY_VALUE_SUPPLIER = "importSecurityFieldSupplier";
 
    public static final String FIELD_ARCHIVE_FILE_ENABLED     = "archiveFileEnabled";
    public static final String FIELD_ARCHIVE_TABLE_NAME       = "archiveTableName";
@@ -93,6 +98,7 @@ public class FilesystemImporterStep implements BackendStep
 
    public static final String FIELD_UPDATE_FILE_IF_NAME_EXISTS = "updateFileIfNameExists";
 
+   private Function<QRecord, Serializable> securitySupplier = null;
 
 
    /*******************************************************************************
@@ -267,9 +273,34 @@ public class FilesystemImporterStep implements BackendStep
     *******************************************************************************/
    private void addSecurityValue(RunBackendStepInput runBackendStepInput, QRecord record)
    {
-      String       securityField = runBackendStepInput.getValueString(FIELD_IMPORT_SECURITY_FIELD_NAME);
+      String securityField = runBackendStepInput.getValueString(FIELD_IMPORT_SECURITY_FIELD_NAME);
-      Serializable securityValue = runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_FIELD_VALUE);
 
+      /////////////////////////////////////////////////////////////
+      // if we're using a security supplier function, load it up //
+      /////////////////////////////////////////////////////////////
+      QCodeReference securitySupplierReference = (QCodeReference) runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_VALUE_SUPPLIER);
+      try
+      {
+         if(securitySupplierReference != null && securitySupplier == null)
+         {
+            securitySupplier = QCodeLoader.getAdHoc(Function.class, securitySupplierReference);
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QRuntimeException("Error loading Security Supplier Function from QCodeReference [" + securitySupplierReference + "]", e));
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // either get the security value from the supplier, or the field value field's value //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      Serializable securityValue = securitySupplier != null
+         ? securitySupplier.apply(record)
+         : runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_FIELD_VALUE);
+
+      ////////////////////////////////////////////////////////////////////
+      // if we have a field name and a value, then add it to the record //
+      ////////////////////////////////////////////////////////////////////
       if(StringUtils.hasContent(securityField) && securityValue != null)
       {
          record.setValue(securityField, securityValue);

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java

@@ -23,16 +23,20 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
 
 
 import java.io.File;
+import java.io.Serializable;
 import java.time.LocalDateTime;
+import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@@ -261,4 +265,57 @@ class FilesystemImporterStepTest extends FilesystemActionTest
       assertEquals(47, recordRecord.getValue("customerId"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSecuritySupplier() throws QException
+   {
+      //////////////////////////////////////////////
+      // Add a security name/value to our process //
+      //////////////////////////////////////////////
+      QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME)).findFirst().get().setDefaultValue("customerId");
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_VALUE_SUPPLIER)).findFirst().get().setDefaultValue(new QCodeReference(SecuritySupplier.class));
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      // re-validate our instance now that we have that code-reference in place for the security supplier //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().setHasBeenValidated(null);
+      new QInstanceValidator().validate(QContext.getQInstance());
+
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      // assert the security field gets its value on both the importFile & importRecord records //
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      String  importBaseName = "personImporter";
+      QRecord fileRecord     = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(1701, fileRecord.getValue("customerId"));
+
+      QRecord recordRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(1701, recordRecord.getValue("customerId"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class SecuritySupplier implements Function<QRecord, Serializable>
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public Serializable apply(QRecord qRecord)
+      {
+         return (1701);
+      }
+   }
+
 }