CE-781 Fix archivePath as field on table; set maxRows 100 on child-widget; always archive files; allow security name/value; significant tests on importer step

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java

@@ -177,6 +177,7 @@ public class FilesystemImporterMetaDataTemplate
       return ChildRecordListRenderer.widgetMetaDataBuilder(join)
          .withName(join.getName())
          .withLabel("Import Records")
+         .withMaxRows(100)
          .withCanAddChildRecord(false)
          .getWidgetMetaData();
    }
@@ -215,10 +216,11 @@ public class FilesystemImporterMetaDataTemplate
 
          .withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend)))
          .withField(new QFieldMetaData("sourceFileName", QFieldType.STRING))
+         .withField(new QFieldMetaData("archivedPath", QFieldType.STRING))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
 
-         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName")))
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName", "archivedPath")))
          .withSection(new QFieldSection("records", new QIcon().withName("power_input"), Tier.T2).withWidgetName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX))
          .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
 
 
+import java.io.Serializable;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@@ -59,6 +60,8 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
                .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, QFieldType.BOOLEAN).withDefaultValue(false))
                .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, QFieldType.STRING))
                .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_PATH, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, QFieldType.STRING))
             )));
    }
 
@@ -161,4 +164,26 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldName(String securityFieldName)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, securityFieldName);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldValue(Serializable securityFieldValue)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, securityFieldValue);
+      return (this);
+   }
+
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java

@@ -28,10 +28,10 @@ import java.io.InputStream;
 import java.io.Serializable;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 import java.util.UUID;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
@@ -56,8 +56,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -81,6 +83,9 @@ public class FilesystemImporterStep implements BackendStep
    public static final String FIELD_IMPORT_FILE_TABLE   = "importFileTable";
    public static final String FIELD_IMPORT_RECORD_TABLE = "importRecordTable";
 
+   public static final String FIELD_IMPORT_SECURITY_FIELD_NAME  = "importSecurityFieldName";
+   public static final String FIELD_IMPORT_SECURITY_FIELD_VALUE = "importSecurityFieldValue";
+
    public static final String FIELD_ARCHIVE_FILE_ENABLED     = "archiveFileEnabled";
    public static final String FIELD_ARCHIVE_TABLE_NAME       = "archiveTableName";
    public static final String FIELD_ARCHIVE_PATH             = "archivePath";
@@ -173,6 +178,7 @@ public class FilesystemImporterStep implements BackendStep
                else
                {
                   LOG.debug("Skipping already-imported file", logPair("fileName", sourceFileName));
+                  removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName);
                   continue;
                }
             }
@@ -198,11 +204,12 @@ public class FilesystemImporterStep implements BackendStep
                /////////////////////////////////
                LOG.info("Syncing file [" + sourceFileName + "]");
                QRecord importFileRecord = new QRecord()
-                  // todo - how to get clientId in here?
                   .withValue("id", idToUpdate)
                   .withValue("sourceFileName", sourceFileName)
                   .withValue("archivedPath", archivedPath);
 
+               addSecurityValue(runBackendStepInput, importFileRecord);
+
                //////////////////////////////////////
                // build child importRecord records //
                //////////////////////////////////////
@@ -232,11 +239,7 @@ public class FilesystemImporterStep implements BackendStep
             // if we are interrupted between the commit & the delete, then the file will be found again, //
             // and we'll either skip it or do an update, based on FIELD_UPDATE_FILE_IF_NAME_EXISTS flag  //
             ///////////////////////////////////////////////////////////////////////////////////////////////
-            if(removeFileAfterImport)
+            removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName);
-            {
-               String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
-               sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
-            }
          }
          catch(Exception e)
          {
@@ -258,6 +261,37 @@ public class FilesystemImporterStep implements BackendStep
 
 
 
+   /*******************************************************************************
+    ** if the process is configured w/ a security field & value, set it on the import
+    ** File & Record records.
+    *******************************************************************************/
+   private void addSecurityValue(RunBackendStepInput runBackendStepInput, QRecord record)
+   {
+      String       securityField = runBackendStepInput.getValueString(FIELD_IMPORT_SECURITY_FIELD_NAME);
+      Serializable securityValue = runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_FIELD_VALUE);
+
+      if(StringUtils.hasContent(securityField) && securityValue != null)
+      {
+         record.setValue(securityField, securityValue);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException
+   {
+      if(removeFileAfterImport)
+      {
+         String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
+         sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -288,6 +322,8 @@ public class FilesystemImporterStep implements BackendStep
          + File.separator + now.getMonth()
          + File.separator + UUID.randomUUID()
          + "-" + sourceFileName.replaceAll(".*" + File.separator, "");
+      path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
+
       LOG.info("Archiving file", logPair("path", path));
       archiveActionBase.writeFile(archiveBackend, path, bytes);
 
@@ -325,15 +361,15 @@ public class FilesystemImporterStep implements BackendStep
          default -> throw (new QException("Unexpected file format: " + fileFormat));
       };
 
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////////
-      // now, wrap those records with the fields of the importRecord table, putting the unknown fields in a blob together //
+      // now, add some fields that we know about to those records, for returning //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////////
       List<QRecord> importRecordList = new ArrayList<>();
       int           recordNo         = 1;
       for(QRecord record : contentRecords)
       {
          record.setValue("recordNo", recordNo++);
-         // todo - client_id??
+         addSecurityValue(runBackendStepInput, record);
 
          importRecordList.add(record);
       }
@@ -348,8 +384,12 @@ public class FilesystemImporterStep implements BackendStep
     *******************************************************************************/
    private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
    {
-      List<F>        files = actionBase.listFiles(table, backend);
+      List<F> files = actionBase.listFiles(table, backend);
-      Map<String, F> rs    = new LinkedHashMap<>();
+
+      /////////////////////////////////////////////////////
+      // use a tree map, so files will be sorted by name //
+      /////////////////////////////////////////////////////
+      Map<String, F> rs = new TreeMap<>();
 
       for(F file : files)
       {

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java

@@ -154,6 +154,18 @@ public class TestUtils
       qInstance.addTable(defineMockPersonTable());
       qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
 
+      definePersonCsvImporter(qInstance);
+
+      return (qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void definePersonCsvImporter(QInstance qInstance)
+   {
       String importBaseName = "personImporter";
       FilesystemImporterProcessMetaDataBuilder filesystemImporterProcessMetaDataBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
          .withSourceTableName(TABLE_NAME_PERSON_LOCAL_FS_CSV)
@@ -164,10 +176,7 @@ public class TestUtils
          .withName(LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
 
       FilesystemImporterMetaDataTemplate filesystemImporterMetaDataTemplate = new FilesystemImporterMetaDataTemplate(qInstance, importBaseName, BACKEND_NAME_MEMORY, filesystemImporterProcessMetaDataBuilder, table -> table.withAuditRules(QAuditRules.defaultInstanceLevelNone()));
-
       filesystemImporterMetaDataTemplate.addToInstance(qInstance);
-
-      return (qInstance);
    }
 
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java

@@ -130,7 +130,7 @@ public class FilesystemActionTest extends BaseTest
    /*******************************************************************************
     ** Write some data files into the directory for the filesystem module.
     *******************************************************************************/
-   private void writePersonCSVFiles(File baseDirectory) throws IOException
+   protected void writePersonCSVFiles(File baseDirectory) throws IOException
    {
       String fullPath = baseDirectory.getAbsolutePath();
       if(TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
@@ -42,7 +43,7 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.fail;
 
 
 /*******************************************************************************
@@ -62,7 +63,7 @@ class FilesystemImporterStepTest extends FilesystemActionTest
     **
     *******************************************************************************/
    @AfterEach
-   public void filesystemBaseAfterEach() throws Exception
+   public void afterEach() throws Exception
    {
       MemoryRecordStore.getInstance().reset();
    }
@@ -75,6 +76,14 @@ class FilesystemImporterStepTest extends FilesystemActionTest
    @Test
    void test() throws QException
    {
+      /////////////////////////////////////////////////////
+      // make sure we see 2 source files before we begin //
+      /////////////////////////////////////////////////////
+      FilesystemBackendMetaData backend   = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
+      String                    basePath  = backend.getBasePath();
+      File                      sourceDir = new File(basePath + "/persons-csv/");
+      assertEquals(2, listOrFail(sourceDir).length);
+
       RunProcessInput runProcessInput = new RunProcessInput();
       runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
       new RunProcessAction().execute(runProcessInput);
@@ -90,25 +99,166 @@ class FilesystemImporterStepTest extends FilesystemActionTest
       JSONObject values = new JSONObject(record.getValueString("values"));
       assertEquals("John", values.get("firstName"));
 
-      FilesystemBackendMetaData backend  = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
-      String                    basePath = backend.getBasePath();
-      System.out.println(basePath);
-
       ///////////////////////////////////////////
       // make sure 2 archive files got created //
       ///////////////////////////////////////////
-      LocalDateTime now   = LocalDateTime.now();
+      LocalDateTime now = LocalDateTime.now();
-      File[]        files = new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth()).listFiles();
+      assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length);
-      assertNotNull(files);
+
-      assertEquals(2, files.length);
+      ////////////////////////////////////////////
+      // make sure the source files got deleted //
+      ////////////////////////////////////////////
+      assertEquals(0, listOrFail(sourceDir).length);
    }
 
-   // todo - test json
 
-   // todo - test no files found
 
-   // todo - confirm delete happens?
+   /*******************************************************************************
+    ** do a listFiles, but fail properly if it returns null (so IJ won't warn all the time)
+    *******************************************************************************/
+   private static File[] listOrFail(File dir)
+   {
+      File[] files = dir.listFiles();
+      if(files == null)
+      {
+         fail("Null result when listing directory: " + dir);
+      }
+      return (files);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testJSON() throws QException
+   {
+      ////////////////////////////////////////////////////////////////////
+      // adjust the process to use the JSON file table, and JSON format //
+      ////////////////////////////////////////////////////////////////////
+      QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_SOURCE_TABLE)).findFirst().get().setDefaultValue(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_FILE_FORMAT)).findFirst().get().setDefaultValue("json");
+
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      String importBaseName = "personImporter";
+      assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(3, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+
+      QRecord record = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(1, record.getValue("importFileId"));
+      assertEquals("John", record.getValue("firstName"));
+      assertThat(record.getValue("values")).isInstanceOf(String.class);
+      JSONObject values = new JSONObject(record.getValueString("values"));
+      assertEquals("John", values.get("firstName"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNoFilesFound() throws Exception
+   {
+      cleanFilesystem();
+
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      String importBaseName = "personImporter";
+      assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+   }
 
    // todo - updates?
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDuplicateFileNameNonUpdate() throws Exception
+   {
+      FilesystemBackendMetaData backend   = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
+      String                    basePath  = backend.getBasePath();
+      File                      sourceDir = new File(basePath + "/persons-csv/");
+
+      /////////////////////////////////////////////////////////////////
+      // run the process once - assert how many records got inserted //
+      /////////////////////////////////////////////////////////////////
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      String importBaseName = "personImporter";
+      assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+
+      ///////////////////////////////////////////////////////
+      // put the source files back - assert they are there //
+      ///////////////////////////////////////////////////////
+      writePersonCSVFiles(new File(basePath));
+      assertEquals(2, listOrFail(sourceDir).length);
+
+      ////////////////////////
+      // re-run the process //
+      ////////////////////////
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      ////////////////////////////////////////
+      // make sure no new records are built //
+      ////////////////////////////////////////
+      assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+
+      /////////////////////////////////////////////////
+      // make sure no new archive files were created //
+      /////////////////////////////////////////////////
+      LocalDateTime now = LocalDateTime.now();
+      assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length);
+
+      ////////////////////////////////////////////
+      // make sure the source files got deleted //
+      ////////////////////////////////////////////
+      assertEquals(0, listOrFail(sourceDir).length);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSecurityKey() throws QException
+   {
+      //////////////////////////////////////////////
+      // Add a security name/value to our process //
+      //////////////////////////////////////////////
+      QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME)).findFirst().get().setDefaultValue("customerId");
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE)).findFirst().get().setDefaultValue(47);
+
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      // assert the security field gets its value on both the importFile & importRecord records //
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      String  importBaseName = "personImporter";
+      QRecord fileRecord     = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(47, fileRecord.getValue("customerId"));
+
+      QRecord recordRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(47, recordRecord.getValue("customerId"));
+   }
+
 }