diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java index b23898cf..760ffd4a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java @@ -177,6 +177,7 @@ public class FilesystemImporterMetaDataTemplate return ChildRecordListRenderer.widgetMetaDataBuilder(join) .withName(join.getName()) .withLabel("Import Records") + .withMaxRows(100) .withCanAddChildRecord(false) .getWidgetMetaData(); } @@ -215,10 +216,11 @@ public class FilesystemImporterMetaDataTemplate .withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend))) .withField(new QFieldMetaData("sourceFileName", QFieldType.STRING)) + .withField(new QFieldMetaData("archivedPath", QFieldType.STRING)) .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) - .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName"))) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName", "archivedPath"))) .withSection(new QFieldSection("records", new QIcon().withName("power_input"), Tier.T2).withWidgetName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX)) .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java index 7e078f16..d90992a2 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer; +import java.io.Serializable; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -59,6 +60,8 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, QFieldType.BOOLEAN).withDefaultValue(false)) .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, QFieldType.STRING)) .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_PATH, QFieldType.STRING)) + .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, QFieldType.STRING)) + .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, QFieldType.STRING)) ))); } @@ -161,4 +164,26 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldName(String securityFieldName) + { + setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, securityFieldName); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldValue(Serializable securityFieldValue) + { + setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, securityFieldValue); + return (this); + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java index c2b82495..5537c9fd 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java @@ -28,10 +28,10 @@ import java.io.InputStream; import java.io.Serializable; import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; @@ -56,8 +56,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -81,6 +83,9 @@ public class FilesystemImporterStep implements BackendStep public static final String FIELD_IMPORT_FILE_TABLE = "importFileTable"; public static final String FIELD_IMPORT_RECORD_TABLE = "importRecordTable"; + public static final String FIELD_IMPORT_SECURITY_FIELD_NAME = "importSecurityFieldName"; + public static final String FIELD_IMPORT_SECURITY_FIELD_VALUE = "importSecurityFieldValue"; + public static final String FIELD_ARCHIVE_FILE_ENABLED = "archiveFileEnabled"; public static final String FIELD_ARCHIVE_TABLE_NAME = "archiveTableName"; public static final String FIELD_ARCHIVE_PATH = "archivePath"; @@ -173,6 +178,7 @@ public class FilesystemImporterStep implements BackendStep else { LOG.debug("Skipping already-imported file", logPair("fileName", sourceFileName)); + removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName); continue; } } @@ -198,11 +204,12 @@ public class FilesystemImporterStep implements BackendStep ///////////////////////////////// LOG.info("Syncing file [" + sourceFileName + "]"); QRecord importFileRecord = new QRecord() - // todo - how to get clientId in here? .withValue("id", idToUpdate) .withValue("sourceFileName", sourceFileName) .withValue("archivedPath", archivedPath); + addSecurityValue(runBackendStepInput, importFileRecord); + ////////////////////////////////////// // build child importRecord records // ////////////////////////////////////// @@ -232,11 +239,7 @@ public class FilesystemImporterStep implements BackendStep // if we are interrupted between the commit & the delete, then the file will be found again, // // and we'll either skip it or do an update, based on FIELD_UPDATE_FILE_IF_NAME_EXISTS flag // /////////////////////////////////////////////////////////////////////////////////////////////// - if(removeFileAfterImport) - { - String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend); - sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName); - } + removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName); } catch(Exception e) { @@ -258,6 +261,37 @@ public class FilesystemImporterStep implements BackendStep + /******************************************************************************* + ** if the process is configured w/ a security field & value, set it on the import + ** File & Record records. + *******************************************************************************/ + private void addSecurityValue(RunBackendStepInput runBackendStepInput, QRecord record) + { + String securityField = runBackendStepInput.getValueString(FIELD_IMPORT_SECURITY_FIELD_NAME); + Serializable securityValue = runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_FIELD_VALUE); + + if(StringUtils.hasContent(securityField) && securityValue != null) + { + record.setValue(securityField, securityValue); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException + { + if(removeFileAfterImport) + { + String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend); + sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -288,6 +322,8 @@ public class FilesystemImporterStep implements BackendStep + File.separator + now.getMonth() + File.separator + UUID.randomUUID() + "-" + sourceFileName.replaceAll(".*" + File.separator, ""); + path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path); + LOG.info("Archiving file", logPair("path", path)); archiveActionBase.writeFile(archiveBackend, path, bytes); @@ -325,15 +361,15 @@ public class FilesystemImporterStep implements BackendStep default -> throw (new QException("Unexpected file format: " + fileFormat)); }; - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // now, wrap those records with the fields of the importRecord table, putting the unknown fields in a blob together // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////// + // now, add some fields that we know about to those records, for returning // + ///////////////////////////////////////////////////////////////////////////// List importRecordList = new ArrayList<>(); int recordNo = 1; for(QRecord record : contentRecords) { record.setValue("recordNo", recordNo++); - // todo - client_id?? + addSecurityValue(runBackendStepInput, record); importRecordList.add(record); } @@ -348,8 +384,12 @@ public class FilesystemImporterStep implements BackendStep *******************************************************************************/ private Map getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend) throws QException { - List files = actionBase.listFiles(table, backend); - Map rs = new LinkedHashMap<>(); + List files = actionBase.listFiles(table, backend); + + ///////////////////////////////////////////////////// + // use a tree map, so files will be sorted by name // + ///////////////////////////////////////////////////// + Map rs = new TreeMap<>(); for(F file : files) { diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index 7696c6b3..68c99d28 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -154,6 +154,18 @@ public class TestUtils qInstance.addTable(defineMockPersonTable()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); + definePersonCsvImporter(qInstance); + + return (qInstance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void definePersonCsvImporter(QInstance qInstance) + { String importBaseName = "personImporter"; FilesystemImporterProcessMetaDataBuilder filesystemImporterProcessMetaDataBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder() .withSourceTableName(TABLE_NAME_PERSON_LOCAL_FS_CSV) @@ -164,10 +176,7 @@ public class TestUtils .withName(LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME); FilesystemImporterMetaDataTemplate filesystemImporterMetaDataTemplate = new FilesystemImporterMetaDataTemplate(qInstance, importBaseName, BACKEND_NAME_MEMORY, filesystemImporterProcessMetaDataBuilder, table -> table.withAuditRules(QAuditRules.defaultInstanceLevelNone())); - filesystemImporterMetaDataTemplate.addToInstance(qInstance); - - return (qInstance); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 81dac2ec..243f56c6 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -130,7 +130,7 @@ public class FilesystemActionTest extends BaseTest /******************************************************************************* ** Write some data files into the directory for the filesystem module. *******************************************************************************/ - private void writePersonCSVFiles(File baseDirectory) throws IOException + protected void writePersonCSVFiles(File baseDirectory) throws IOException { String fullPath = baseDirectory.getAbsolutePath(); if(TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java index 794e0e15..89801686 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java @@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest; @@ -42,7 +43,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; /******************************************************************************* @@ -62,7 +63,7 @@ class FilesystemImporterStepTest extends FilesystemActionTest ** *******************************************************************************/ @AfterEach - public void filesystemBaseAfterEach() throws Exception + public void afterEach() throws Exception { MemoryRecordStore.getInstance().reset(); } @@ -75,6 +76,14 @@ class FilesystemImporterStepTest extends FilesystemActionTest @Test void test() throws QException { + ///////////////////////////////////////////////////// + // make sure we see 2 source files before we begin // + ///////////////////////////////////////////////////// + FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS); + String basePath = backend.getBasePath(); + File sourceDir = new File(basePath + "/persons-csv/"); + assertEquals(2, listOrFail(sourceDir).length); + RunProcessInput runProcessInput = new RunProcessInput(); runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME); new RunProcessAction().execute(runProcessInput); @@ -90,25 +99,166 @@ class FilesystemImporterStepTest extends FilesystemActionTest JSONObject values = new JSONObject(record.getValueString("values")); assertEquals("John", values.get("firstName")); - FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS); - String basePath = backend.getBasePath(); - System.out.println(basePath); - /////////////////////////////////////////// // make sure 2 archive files got created // /////////////////////////////////////////// - LocalDateTime now = LocalDateTime.now(); - File[] files = new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth()).listFiles(); - assertNotNull(files); - assertEquals(2, files.length); + LocalDateTime now = LocalDateTime.now(); + assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length); + + //////////////////////////////////////////// + // make sure the source files got deleted // + //////////////////////////////////////////// + assertEquals(0, listOrFail(sourceDir).length); } - // todo - test json - // todo - test no files found - // todo - confirm delete happens? + /******************************************************************************* + ** do a listFiles, but fail properly if it returns null (so IJ won't warn all the time) + *******************************************************************************/ + private static File[] listOrFail(File dir) + { + File[] files = dir.listFiles(); + if(files == null) + { + fail("Null result when listing directory: " + dir); + } + return (files); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testJSON() throws QException + { + //////////////////////////////////////////////////////////////////// + // adjust the process to use the JSON file table, and JSON format // + //////////////////////////////////////////////////////////////////// + QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME); + process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_SOURCE_TABLE)).findFirst().get().setDefaultValue(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); + process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_FILE_FORMAT)).findFirst().get().setDefaultValue("json"); + + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME); + new RunProcessAction().execute(runProcessInput); + + String importBaseName = "personImporter"; + assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount()); + assertEquals(3, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount()); + + QRecord record = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1)); + assertEquals(1, record.getValue("importFileId")); + assertEquals("John", record.getValue("firstName")); + assertThat(record.getValue("values")).isInstanceOf(String.class); + JSONObject values = new JSONObject(record.getValueString("values")); + assertEquals("John", values.get("firstName")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNoFilesFound() throws Exception + { + cleanFilesystem(); + + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME); + new RunProcessAction().execute(runProcessInput); + + String importBaseName = "personImporter"; + assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount()); + assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount()); + } // todo - updates? + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDuplicateFileNameNonUpdate() throws Exception + { + FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS); + String basePath = backend.getBasePath(); + File sourceDir = new File(basePath + "/persons-csv/"); + + ///////////////////////////////////////////////////////////////// + // run the process once - assert how many records got inserted // + ///////////////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME); + new RunProcessAction().execute(runProcessInput); + + String importBaseName = "personImporter"; + assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount()); + assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount()); + + /////////////////////////////////////////////////////// + // put the source files back - assert they are there // + /////////////////////////////////////////////////////// + writePersonCSVFiles(new File(basePath)); + assertEquals(2, listOrFail(sourceDir).length); + + //////////////////////// + // re-run the process // + //////////////////////// + runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME); + new RunProcessAction().execute(runProcessInput); + + //////////////////////////////////////// + // make sure no new records are built // + //////////////////////////////////////// + assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount()); + assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount()); + + ///////////////////////////////////////////////// + // make sure no new archive files were created // + ///////////////////////////////////////////////// + LocalDateTime now = LocalDateTime.now(); + assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length); + + //////////////////////////////////////////// + // make sure the source files got deleted // + //////////////////////////////////////////// + assertEquals(0, listOrFail(sourceDir).length); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSecurityKey() throws QException + { + ////////////////////////////////////////////// + // Add a security name/value to our process // + ////////////////////////////////////////////// + QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME); + process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME)).findFirst().get().setDefaultValue("customerId"); + process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE)).findFirst().get().setDefaultValue(47); + + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME); + new RunProcessAction().execute(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////// + // assert the security field gets its value on both the importFile & importRecord records // + //////////////////////////////////////////////////////////////////////////////////////////// + String importBaseName = "personImporter"; + QRecord fileRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX).withPrimaryKey(1)); + assertEquals(47, fileRecord.getValue("customerId")); + + QRecord recordRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1)); + assertEquals(47, recordRecord.getValue("customerId")); + } + } \ No newline at end of file