mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 05:01:07 +00:00
CE-781 Fix archivePath as field on table; set maxRows 100 on child-widget; always archive files; allow security name/value; significant tests on importer step
This commit is contained in:
@ -177,6 +177,7 @@ public class FilesystemImporterMetaDataTemplate
|
||||
return ChildRecordListRenderer.widgetMetaDataBuilder(join)
|
||||
.withName(join.getName())
|
||||
.withLabel("Import Records")
|
||||
.withMaxRows(100)
|
||||
.withCanAddChildRecord(false)
|
||||
.getWidgetMetaData();
|
||||
}
|
||||
@ -215,10 +216,11 @@ public class FilesystemImporterMetaDataTemplate
|
||||
|
||||
.withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend)))
|
||||
.withField(new QFieldMetaData("sourceFileName", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("archivedPath", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
|
||||
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName")))
|
||||
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName", "archivedPath")))
|
||||
.withSection(new QFieldSection("records", new QIcon().withName("power_input"), Tier.T2).withWidgetName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX))
|
||||
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
@ -59,6 +60,8 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
|
||||
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, QFieldType.STRING))
|
||||
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_PATH, QFieldType.STRING))
|
||||
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, QFieldType.STRING))
|
||||
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, QFieldType.STRING))
|
||||
)));
|
||||
}
|
||||
|
||||
@ -161,4 +164,26 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldName(String securityFieldName)
|
||||
{
|
||||
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, securityFieldName);
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldValue(Serializable securityFieldValue)
|
||||
{
|
||||
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, securityFieldValue);
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -28,10 +28,10 @@ import java.io.InputStream;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.TreeMap;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
@ -56,8 +56,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
@ -81,6 +83,9 @@ public class FilesystemImporterStep implements BackendStep
|
||||
public static final String FIELD_IMPORT_FILE_TABLE = "importFileTable";
|
||||
public static final String FIELD_IMPORT_RECORD_TABLE = "importRecordTable";
|
||||
|
||||
public static final String FIELD_IMPORT_SECURITY_FIELD_NAME = "importSecurityFieldName";
|
||||
public static final String FIELD_IMPORT_SECURITY_FIELD_VALUE = "importSecurityFieldValue";
|
||||
|
||||
public static final String FIELD_ARCHIVE_FILE_ENABLED = "archiveFileEnabled";
|
||||
public static final String FIELD_ARCHIVE_TABLE_NAME = "archiveTableName";
|
||||
public static final String FIELD_ARCHIVE_PATH = "archivePath";
|
||||
@ -173,6 +178,7 @@ public class FilesystemImporterStep implements BackendStep
|
||||
else
|
||||
{
|
||||
LOG.debug("Skipping already-imported file", logPair("fileName", sourceFileName));
|
||||
removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@ -198,11 +204,12 @@ public class FilesystemImporterStep implements BackendStep
|
||||
/////////////////////////////////
|
||||
LOG.info("Syncing file [" + sourceFileName + "]");
|
||||
QRecord importFileRecord = new QRecord()
|
||||
// todo - how to get clientId in here?
|
||||
.withValue("id", idToUpdate)
|
||||
.withValue("sourceFileName", sourceFileName)
|
||||
.withValue("archivedPath", archivedPath);
|
||||
|
||||
addSecurityValue(runBackendStepInput, importFileRecord);
|
||||
|
||||
//////////////////////////////////////
|
||||
// build child importRecord records //
|
||||
//////////////////////////////////////
|
||||
@ -232,11 +239,7 @@ public class FilesystemImporterStep implements BackendStep
|
||||
// if we are interrupted between the commit & the delete, then the file will be found again, //
|
||||
// and we'll either skip it or do an update, based on FIELD_UPDATE_FILE_IF_NAME_EXISTS flag //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(removeFileAfterImport)
|
||||
{
|
||||
String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
|
||||
sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
|
||||
}
|
||||
removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
@ -258,6 +261,37 @@ public class FilesystemImporterStep implements BackendStep
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** if the process is configured w/ a security field & value, set it on the import
|
||||
** File & Record records.
|
||||
*******************************************************************************/
|
||||
private void addSecurityValue(RunBackendStepInput runBackendStepInput, QRecord record)
|
||||
{
|
||||
String securityField = runBackendStepInput.getValueString(FIELD_IMPORT_SECURITY_FIELD_NAME);
|
||||
Serializable securityValue = runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_FIELD_VALUE);
|
||||
|
||||
if(StringUtils.hasContent(securityField) && securityValue != null)
|
||||
{
|
||||
record.setValue(securityField, securityValue);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException
|
||||
{
|
||||
if(removeFileAfterImport)
|
||||
{
|
||||
String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
|
||||
sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -288,6 +322,8 @@ public class FilesystemImporterStep implements BackendStep
|
||||
+ File.separator + now.getMonth()
|
||||
+ File.separator + UUID.randomUUID()
|
||||
+ "-" + sourceFileName.replaceAll(".*" + File.separator, "");
|
||||
path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
|
||||
|
||||
LOG.info("Archiving file", logPair("path", path));
|
||||
archiveActionBase.writeFile(archiveBackend, path, bytes);
|
||||
|
||||
@ -325,15 +361,15 @@ public class FilesystemImporterStep implements BackendStep
|
||||
default -> throw (new QException("Unexpected file format: " + fileFormat));
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// now, wrap those records with the fields of the importRecord table, putting the unknown fields in a blob together //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
// now, add some fields that we know about to those records, for returning //
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
List<QRecord> importRecordList = new ArrayList<>();
|
||||
int recordNo = 1;
|
||||
for(QRecord record : contentRecords)
|
||||
{
|
||||
record.setValue("recordNo", recordNo++);
|
||||
// todo - client_id??
|
||||
addSecurityValue(runBackendStepInput, record);
|
||||
|
||||
importRecordList.add(record);
|
||||
}
|
||||
@ -349,7 +385,11 @@ public class FilesystemImporterStep implements BackendStep
|
||||
private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
|
||||
{
|
||||
List<F> files = actionBase.listFiles(table, backend);
|
||||
Map<String, F> rs = new LinkedHashMap<>();
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// use a tree map, so files will be sorted by name //
|
||||
/////////////////////////////////////////////////////
|
||||
Map<String, F> rs = new TreeMap<>();
|
||||
|
||||
for(F file : files)
|
||||
{
|
||||
|
||||
@ -154,6 +154,18 @@ public class TestUtils
|
||||
qInstance.addTable(defineMockPersonTable());
|
||||
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
|
||||
|
||||
definePersonCsvImporter(qInstance);
|
||||
|
||||
return (qInstance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void definePersonCsvImporter(QInstance qInstance)
|
||||
{
|
||||
String importBaseName = "personImporter";
|
||||
FilesystemImporterProcessMetaDataBuilder filesystemImporterProcessMetaDataBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
|
||||
.withSourceTableName(TABLE_NAME_PERSON_LOCAL_FS_CSV)
|
||||
@ -164,10 +176,7 @@ public class TestUtils
|
||||
.withName(LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
|
||||
|
||||
FilesystemImporterMetaDataTemplate filesystemImporterMetaDataTemplate = new FilesystemImporterMetaDataTemplate(qInstance, importBaseName, BACKEND_NAME_MEMORY, filesystemImporterProcessMetaDataBuilder, table -> table.withAuditRules(QAuditRules.defaultInstanceLevelNone()));
|
||||
|
||||
filesystemImporterMetaDataTemplate.addToInstance(qInstance);
|
||||
|
||||
return (qInstance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -130,7 +130,7 @@ public class FilesystemActionTest extends BaseTest
|
||||
/*******************************************************************************
|
||||
** Write some data files into the directory for the filesystem module.
|
||||
*******************************************************************************/
|
||||
private void writePersonCSVFiles(File baseDirectory) throws IOException
|
||||
protected void writePersonCSVFiles(File baseDirectory) throws IOException
|
||||
{
|
||||
String fullPath = baseDirectory.getAbsolutePath();
|
||||
if(TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
|
||||
|
||||
@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
|
||||
@ -42,7 +43,7 @@ import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -62,7 +63,7 @@ class FilesystemImporterStepTest extends FilesystemActionTest
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
public void filesystemBaseAfterEach() throws Exception
|
||||
public void afterEach() throws Exception
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
}
|
||||
@ -75,6 +76,14 @@ class FilesystemImporterStepTest extends FilesystemActionTest
|
||||
@Test
|
||||
void test() throws QException
|
||||
{
|
||||
/////////////////////////////////////////////////////
|
||||
// make sure we see 2 source files before we begin //
|
||||
/////////////////////////////////////////////////////
|
||||
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
|
||||
String basePath = backend.getBasePath();
|
||||
File sourceDir = new File(basePath + "/persons-csv/");
|
||||
assertEquals(2, listOrFail(sourceDir).length);
|
||||
|
||||
RunProcessInput runProcessInput = new RunProcessInput();
|
||||
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
|
||||
new RunProcessAction().execute(runProcessInput);
|
||||
@ -90,25 +99,166 @@ class FilesystemImporterStepTest extends FilesystemActionTest
|
||||
JSONObject values = new JSONObject(record.getValueString("values"));
|
||||
assertEquals("John", values.get("firstName"));
|
||||
|
||||
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
|
||||
String basePath = backend.getBasePath();
|
||||
System.out.println(basePath);
|
||||
|
||||
///////////////////////////////////////////
|
||||
// make sure 2 archive files got created //
|
||||
///////////////////////////////////////////
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
File[] files = new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth()).listFiles();
|
||||
assertNotNull(files);
|
||||
assertEquals(2, files.length);
|
||||
assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length);
|
||||
|
||||
////////////////////////////////////////////
|
||||
// make sure the source files got deleted //
|
||||
////////////////////////////////////////////
|
||||
assertEquals(0, listOrFail(sourceDir).length);
|
||||
}
|
||||
|
||||
// todo - test json
|
||||
|
||||
// todo - test no files found
|
||||
|
||||
// todo - confirm delete happens?
|
||||
/*******************************************************************************
|
||||
** do a listFiles, but fail properly if it returns null (so IJ won't warn all the time)
|
||||
*******************************************************************************/
|
||||
private static File[] listOrFail(File dir)
|
||||
{
|
||||
File[] files = dir.listFiles();
|
||||
if(files == null)
|
||||
{
|
||||
fail("Null result when listing directory: " + dir);
|
||||
}
|
||||
return (files);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testJSON() throws QException
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// adjust the process to use the JSON file table, and JSON format //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
|
||||
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_SOURCE_TABLE)).findFirst().get().setDefaultValue(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
|
||||
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_FILE_FORMAT)).findFirst().get().setDefaultValue("json");
|
||||
|
||||
RunProcessInput runProcessInput = new RunProcessInput();
|
||||
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
|
||||
new RunProcessAction().execute(runProcessInput);
|
||||
|
||||
String importBaseName = "personImporter";
|
||||
assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
|
||||
assertEquals(3, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
|
||||
|
||||
QRecord record = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
|
||||
assertEquals(1, record.getValue("importFileId"));
|
||||
assertEquals("John", record.getValue("firstName"));
|
||||
assertThat(record.getValue("values")).isInstanceOf(String.class);
|
||||
JSONObject values = new JSONObject(record.getValueString("values"));
|
||||
assertEquals("John", values.get("firstName"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNoFilesFound() throws Exception
|
||||
{
|
||||
cleanFilesystem();
|
||||
|
||||
RunProcessInput runProcessInput = new RunProcessInput();
|
||||
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
|
||||
new RunProcessAction().execute(runProcessInput);
|
||||
|
||||
String importBaseName = "personImporter";
|
||||
assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
|
||||
assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
|
||||
}
|
||||
|
||||
// todo - updates?
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDuplicateFileNameNonUpdate() throws Exception
|
||||
{
|
||||
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
|
||||
String basePath = backend.getBasePath();
|
||||
File sourceDir = new File(basePath + "/persons-csv/");
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// run the process once - assert how many records got inserted //
|
||||
/////////////////////////////////////////////////////////////////
|
||||
RunProcessInput runProcessInput = new RunProcessInput();
|
||||
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
|
||||
new RunProcessAction().execute(runProcessInput);
|
||||
|
||||
String importBaseName = "personImporter";
|
||||
assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
|
||||
assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
|
||||
|
||||
///////////////////////////////////////////////////////
|
||||
// put the source files back - assert they are there //
|
||||
///////////////////////////////////////////////////////
|
||||
writePersonCSVFiles(new File(basePath));
|
||||
assertEquals(2, listOrFail(sourceDir).length);
|
||||
|
||||
////////////////////////
|
||||
// re-run the process //
|
||||
////////////////////////
|
||||
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
|
||||
new RunProcessAction().execute(runProcessInput);
|
||||
|
||||
////////////////////////////////////////
|
||||
// make sure no new records are built //
|
||||
////////////////////////////////////////
|
||||
assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
|
||||
assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// make sure no new archive files were created //
|
||||
/////////////////////////////////////////////////
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length);
|
||||
|
||||
////////////////////////////////////////////
|
||||
// make sure the source files got deleted //
|
||||
////////////////////////////////////////////
|
||||
assertEquals(0, listOrFail(sourceDir).length);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSecurityKey() throws QException
|
||||
{
|
||||
//////////////////////////////////////////////
|
||||
// Add a security name/value to our process //
|
||||
//////////////////////////////////////////////
|
||||
QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
|
||||
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME)).findFirst().get().setDefaultValue("customerId");
|
||||
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE)).findFirst().get().setDefaultValue(47);
|
||||
|
||||
RunProcessInput runProcessInput = new RunProcessInput();
|
||||
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
|
||||
new RunProcessAction().execute(runProcessInput);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// assert the security field gets its value on both the importFile & importRecord records //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
String importBaseName = "personImporter";
|
||||
QRecord fileRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX).withPrimaryKey(1));
|
||||
assertEquals(47, fileRecord.getValue("customerId"));
|
||||
|
||||
QRecord recordRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
|
||||
assertEquals(47, recordRecord.getValue("customerId"));
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user