QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-773 Update for compat. with previous commit, but also, fix all generics and move inputStream into try-with-resources

Browse Source
This commit is contained in:
darin.kelkhoff
2023-12-27 16:11:19 -06:00
parent 940080bc86
commit b805e7645b
1 changed files with 44 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java
View File

@ -59,31 +59,45 @@ public class FilesystemSyncStep implements BackendStep
*******************************************************************************/ *******************************************************************************/
@Override @Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// defer to a private method here, so we can add a type-parameter for that method to use //
// would think we could do that here, but get compiler error, since this method comes from base class //
////////////////////////////////////////////////////////////////////////////////////////////////////////
doRun(runBackendStepInput, runBackendStepOutput);
}
/*******************************************************************************
**
*******************************************************************************/
private <F> void doRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{ {
QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); QTableMetaData archiveTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); QTableMetaData processingTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName()); QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); FilesystemBackendModuleInterface<F> sourceModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); AbstractBaseFilesystemAction<F> sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend); sourceActionBase.preAction(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); Map<String, F> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName()); QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); FilesystemBackendModuleInterface<F> archiveModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); AbstractBaseFilesystemAction<F> archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend); archiveActionBase.preAction(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet(); Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName()); QBackendMetaData processingBackend = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); FilesystemBackendModuleInterface<F> processingModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); AbstractBaseFilesystemAction<F> processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend); processingActionBase.preAction(processingBackend);
Integer maxFilesToSync = runBackendStepInput.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); Integer maxFilesToSync = runBackendStepInput.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
int syncedFileCount = 0; int syncedFileCount = 0;
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet()) for(Map.Entry<String, F> sourceEntry : sourceFiles.entrySet())
{ {
try try
{ {
@ -91,7 +105,8 @@ public class FilesystemSyncStep implements BackendStep
if(!archiveFiles.contains(sourceFileName)) if(!archiveFiles.contains(sourceFileName))
{ {
LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]"); LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()); try(InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()))
{
byte[] bytes = inputStream.readAllBytes(); byte[] bytes = inputStream.readAllBytes();
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend); String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
@ -108,6 +123,7 @@ public class FilesystemSyncStep implements BackendStep
} }
} }
} }
}
catch(Exception e) catch(Exception e)
{ {
LOG.error("Error processing file: " + sourceEntry, e); LOG.error("Error processing file: " + sourceEntry, e);
@ -120,12 +136,12 @@ public class FilesystemSyncStep implements BackendStep
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
private Map<String, Object> getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend) private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
{ {
List<Object> files = actionBase.listFiles(table, backend); List<F> files = actionBase.listFiles(table, backend);
Map<String, Object> rs = new LinkedHashMap<>(); Map<String, F> rs = new LinkedHashMap<>();
for(Object file : files) for(F file : files)
{ {
String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table); String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
rs.put(fileName, file); rs.put(fileName, file);