From 4bc0ab1c4913be7c4d92eccfe85aed959742c725 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 28 Jun 2022 13:49:37 -0500 Subject: [PATCH] QQQ-14 fixes to sync process --- .../actions/AbstractBaseFilesystemAction.java | 5 ++- .../sync/FilesystemSyncFunction.java | 32 +++++++++++-------- .../s3/actions/AbstractS3Action.java | 24 +++++++++++--- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 1df77c93..e5c69153 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -32,7 +32,6 @@ import java.util.function.Function; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -182,7 +181,7 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public QueryResult executeQuery(QueryRequest queryRequest) throws QException { - preAction(queryRequest); + preAction(queryRequest.getBackend()); try { @@ -256,7 +255,7 @@ public abstract class AbstractBaseFilesystemAction ** Method that subclasses can override to add pre-action things (e.g., setting up ** s3 client). *******************************************************************************/ - protected void preAction(AbstractQTableRequest tableRequest) + public void preAction(QBackendMetaData backendMetaData) { ///////////////////////////////////////////////////////////////////// // noop in base class - subclasses can add functionality if needed // diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java index be043bc7..5ca6c732 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java @@ -65,16 +65,22 @@ public class FilesystemSyncFunction implements FunctionBody QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); - QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); - FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); - Map sourceFiles = getFileNames(sourceModule.getActionBase(), sourceTable, sourceBackend); + QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); + FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); + AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); + sourceActionBase.preAction(sourceBackend); + Map sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); - QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); - FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); - Set archiveFiles = getFileNames(archiveModule.getActionBase(), archiveTable, archiveBackend).keySet(); + QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); + FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); + AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); + archiveActionBase.preAction(archiveBackend); + Set archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet(); - QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); - FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); + QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); + FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); + AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); + processingActionBase.preAction(processingBackend); for(Map.Entry sourceEntry : sourceFiles.entrySet()) { @@ -84,14 +90,14 @@ public class FilesystemSyncFunction implements FunctionBody if(!archiveFiles.contains(sourceFileName)) { LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]"); - InputStream inputStream = sourceModule.getActionBase().readFile(sourceEntry.getValue()); + InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()); byte[] bytes = inputStream.readAllBytes(); - String archivePath = archiveModule.getActionBase().getFullBasePath(archiveTable, archiveBackend); - archiveModule.getActionBase().writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); + String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend); + archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); - String processingPath = processingModule.getActionBase().getFullBasePath(processingTable, processingBackend); - processingModule.getActionBase().writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); + String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend); + processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); } } catch(Exception e) diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 281d3583..a7ae421b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -29,7 +29,6 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; @@ -58,9 +57,9 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction