CE-781 Fix archivePath as field on table; set maxRows 100 on child-widget; always archive files; allow security name/value; significant tests on importer step

This commit is contained in:
2024-01-16 10:33:59 -06:00
parent 494f0242ac
commit 3ebc567299
6 changed files with 257 additions and 31 deletions

View File

@ -177,6 +177,7 @@ public class FilesystemImporterMetaDataTemplate
return ChildRecordListRenderer.widgetMetaDataBuilder(join)
.withName(join.getName())
.withLabel("Import Records")
.withMaxRows(100)
.withCanAddChildRecord(false)
.getWidgetMetaData();
}
@ -215,10 +216,11 @@ public class FilesystemImporterMetaDataTemplate
.withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend)))
.withField(new QFieldMetaData("sourceFileName", QFieldType.STRING))
.withField(new QFieldMetaData("archivedPath", QFieldType.STRING))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName")))
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName", "archivedPath")))
.withSection(new QFieldSection("records", new QIcon().withName("power_input"), Tier.T2).withWidgetName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX))
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))

View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@ -59,6 +60,8 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, QFieldType.BOOLEAN).withDefaultValue(false))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_PATH, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, QFieldType.STRING))
)));
}
@ -161,4 +164,26 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldName(String securityFieldName)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, securityFieldName);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldValue(Serializable securityFieldValue)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, securityFieldValue);
return (this);
}
}

View File

@ -28,10 +28,10 @@ import java.io.InputStream;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
@ -56,8 +56,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -81,6 +83,9 @@ public class FilesystemImporterStep implements BackendStep
public static final String FIELD_IMPORT_FILE_TABLE = "importFileTable";
public static final String FIELD_IMPORT_RECORD_TABLE = "importRecordTable";
public static final String FIELD_IMPORT_SECURITY_FIELD_NAME = "importSecurityFieldName";
public static final String FIELD_IMPORT_SECURITY_FIELD_VALUE = "importSecurityFieldValue";
public static final String FIELD_ARCHIVE_FILE_ENABLED = "archiveFileEnabled";
public static final String FIELD_ARCHIVE_TABLE_NAME = "archiveTableName";
public static final String FIELD_ARCHIVE_PATH = "archivePath";
@ -173,6 +178,7 @@ public class FilesystemImporterStep implements BackendStep
else
{
LOG.debug("Skipping already-imported file", logPair("fileName", sourceFileName));
removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName);
continue;
}
}
@ -198,11 +204,12 @@ public class FilesystemImporterStep implements BackendStep
/////////////////////////////////
LOG.info("Syncing file [" + sourceFileName + "]");
QRecord importFileRecord = new QRecord()
// todo - how to get clientId in here?
.withValue("id", idToUpdate)
.withValue("sourceFileName", sourceFileName)
.withValue("archivedPath", archivedPath);
addSecurityValue(runBackendStepInput, importFileRecord);
//////////////////////////////////////
// build child importRecord records //
//////////////////////////////////////
@ -232,11 +239,7 @@ public class FilesystemImporterStep implements BackendStep
// if we are interrupted between the commit & the delete, then the file will be found again, //
// and we'll either skip it or do an update, based on FIELD_UPDATE_FILE_IF_NAME_EXISTS flag //
///////////////////////////////////////////////////////////////////////////////////////////////
if(removeFileAfterImport)
{
String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
}
removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName);
}
catch(Exception e)
{
@ -258,6 +261,37 @@ public class FilesystemImporterStep implements BackendStep
/*******************************************************************************
** if the process is configured w/ a security field & value, set it on the import
** File & Record records.
*******************************************************************************/
private void addSecurityValue(RunBackendStepInput runBackendStepInput, QRecord record)
{
String securityField = runBackendStepInput.getValueString(FIELD_IMPORT_SECURITY_FIELD_NAME);
Serializable securityValue = runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_FIELD_VALUE);
if(StringUtils.hasContent(securityField) && securityValue != null)
{
record.setValue(securityField, securityValue);
}
}
/*******************************************************************************
**
*******************************************************************************/
private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException
{
if(removeFileAfterImport)
{
String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -288,6 +322,8 @@ public class FilesystemImporterStep implements BackendStep
+ File.separator + now.getMonth()
+ File.separator + UUID.randomUUID()
+ "-" + sourceFileName.replaceAll(".*" + File.separator, "");
path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
LOG.info("Archiving file", logPair("path", path));
archiveActionBase.writeFile(archiveBackend, path, bytes);
@ -325,15 +361,15 @@ public class FilesystemImporterStep implements BackendStep
default -> throw (new QException("Unexpected file format: " + fileFormat));
};
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// now, wrap those records with the fields of the importRecord table, putting the unknown fields in a blob together //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////
// now, add some fields that we know about to those records, for returning //
/////////////////////////////////////////////////////////////////////////////
List<QRecord> importRecordList = new ArrayList<>();
int recordNo = 1;
for(QRecord record : contentRecords)
{
record.setValue("recordNo", recordNo++);
// todo - client_id??
addSecurityValue(runBackendStepInput, record);
importRecordList.add(record);
}
@ -348,8 +384,12 @@ public class FilesystemImporterStep implements BackendStep
*******************************************************************************/
private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
{
List<F> files = actionBase.listFiles(table, backend);
Map<String, F> rs = new LinkedHashMap<>();
List<F> files = actionBase.listFiles(table, backend);
/////////////////////////////////////////////////////
// use a tree map, so files will be sorted by name //
/////////////////////////////////////////////////////
Map<String, F> rs = new TreeMap<>();
for(F file : files)
{

View File

@ -154,6 +154,18 @@ public class TestUtils
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
definePersonCsvImporter(qInstance);
return (qInstance);
}
/*******************************************************************************
**
*******************************************************************************/
private static void definePersonCsvImporter(QInstance qInstance)
{
String importBaseName = "personImporter";
FilesystemImporterProcessMetaDataBuilder filesystemImporterProcessMetaDataBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
.withSourceTableName(TABLE_NAME_PERSON_LOCAL_FS_CSV)
@ -164,10 +176,7 @@ public class TestUtils
.withName(LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
FilesystemImporterMetaDataTemplate filesystemImporterMetaDataTemplate = new FilesystemImporterMetaDataTemplate(qInstance, importBaseName, BACKEND_NAME_MEMORY, filesystemImporterProcessMetaDataBuilder, table -> table.withAuditRules(QAuditRules.defaultInstanceLevelNone()));
filesystemImporterMetaDataTemplate.addToInstance(qInstance);
return (qInstance);
}

View File

@ -130,7 +130,7 @@ public class FilesystemActionTest extends BaseTest
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writePersonCSVFiles(File baseDirectory) throws IOException
protected void writePersonCSVFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if(TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)

View File

@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
@ -42,7 +43,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;
/*******************************************************************************
@ -62,7 +63,7 @@ class FilesystemImporterStepTest extends FilesystemActionTest
**
*******************************************************************************/
@AfterEach
public void filesystemBaseAfterEach() throws Exception
public void afterEach() throws Exception
{
MemoryRecordStore.getInstance().reset();
}
@ -75,6 +76,14 @@ class FilesystemImporterStepTest extends FilesystemActionTest
@Test
void test() throws QException
{
/////////////////////////////////////////////////////
// make sure we see 2 source files before we begin //
/////////////////////////////////////////////////////
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
String basePath = backend.getBasePath();
File sourceDir = new File(basePath + "/persons-csv/");
assertEquals(2, listOrFail(sourceDir).length);
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
new RunProcessAction().execute(runProcessInput);
@ -90,25 +99,166 @@ class FilesystemImporterStepTest extends FilesystemActionTest
JSONObject values = new JSONObject(record.getValueString("values"));
assertEquals("John", values.get("firstName"));
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
String basePath = backend.getBasePath();
System.out.println(basePath);
///////////////////////////////////////////
// make sure 2 archive files got created //
///////////////////////////////////////////
LocalDateTime now = LocalDateTime.now();
File[] files = new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth()).listFiles();
assertNotNull(files);
assertEquals(2, files.length);
LocalDateTime now = LocalDateTime.now();
assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length);
////////////////////////////////////////////
// make sure the source files got deleted //
////////////////////////////////////////////
assertEquals(0, listOrFail(sourceDir).length);
}
// todo - test json
// todo - test no files found
// todo - confirm delete happens?
/*******************************************************************************
** do a listFiles, but fail properly if it returns null (so IJ won't warn all the time)
*******************************************************************************/
private static File[] listOrFail(File dir)
{
File[] files = dir.listFiles();
if(files == null)
{
fail("Null result when listing directory: " + dir);
}
return (files);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testJSON() throws QException
{
////////////////////////////////////////////////////////////////////
// adjust the process to use the JSON file table, and JSON format //
////////////////////////////////////////////////////////////////////
QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_SOURCE_TABLE)).findFirst().get().setDefaultValue(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_FILE_FORMAT)).findFirst().get().setDefaultValue("json");
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
new RunProcessAction().execute(runProcessInput);
String importBaseName = "personImporter";
assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
assertEquals(3, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
QRecord record = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
assertEquals(1, record.getValue("importFileId"));
assertEquals("John", record.getValue("firstName"));
assertThat(record.getValue("values")).isInstanceOf(String.class);
JSONObject values = new JSONObject(record.getValueString("values"));
assertEquals("John", values.get("firstName"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testNoFilesFound() throws Exception
{
cleanFilesystem();
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
new RunProcessAction().execute(runProcessInput);
String importBaseName = "personImporter";
assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
}
// todo - updates?
/*******************************************************************************
**
*******************************************************************************/
@Test
void testDuplicateFileNameNonUpdate() throws Exception
{
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
String basePath = backend.getBasePath();
File sourceDir = new File(basePath + "/persons-csv/");
/////////////////////////////////////////////////////////////////
// run the process once - assert how many records got inserted //
/////////////////////////////////////////////////////////////////
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
new RunProcessAction().execute(runProcessInput);
String importBaseName = "personImporter";
assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
///////////////////////////////////////////////////////
// put the source files back - assert they are there //
///////////////////////////////////////////////////////
writePersonCSVFiles(new File(basePath));
assertEquals(2, listOrFail(sourceDir).length);
////////////////////////
// re-run the process //
////////////////////////
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
new RunProcessAction().execute(runProcessInput);
////////////////////////////////////////
// make sure no new records are built //
////////////////////////////////////////
assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
/////////////////////////////////////////////////
// make sure no new archive files were created //
/////////////////////////////////////////////////
LocalDateTime now = LocalDateTime.now();
assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length);
////////////////////////////////////////////
// make sure the source files got deleted //
////////////////////////////////////////////
assertEquals(0, listOrFail(sourceDir).length);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSecurityKey() throws QException
{
//////////////////////////////////////////////
// Add a security name/value to our process //
//////////////////////////////////////////////
QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME)).findFirst().get().setDefaultValue("customerId");
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE)).findFirst().get().setDefaultValue(47);
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
new RunProcessAction().execute(runProcessInput);
////////////////////////////////////////////////////////////////////////////////////////////
// assert the security field gets its value on both the importFile & importRecord records //
////////////////////////////////////////////////////////////////////////////////////////////
String importBaseName = "personImporter";
QRecord fileRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX).withPrimaryKey(1));
assertEquals(47, fileRecord.getValue("customerId"));
QRecord recordRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
assertEquals(47, recordRecord.getValue("customerId"));
}
}