From 2aafb70f43f73b054fe5dda57dc902bbbbe016a6 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 29 Jun 2022 10:21:19 -0500 Subject: [PATCH] QQQ-14 checkpoint, pre-demo --- pom.xml | 2 +- .../base/actions/AbstractBaseFilesystemAction.java | 4 ++-- .../local/actions/AbstractFilesystemAction.java | 6 +++--- .../basic/BasicETLCleanupSourceFilesFunction.java | 13 ++++++++----- .../filesystem/sync/FilesystemSyncFunction.java | 11 ++++++++++- .../filesystem/sync/FilesystemSyncProcess.java | 11 ++++++++--- .../filesystem/s3/actions/AbstractS3Action.java | 7 ++++--- .../s3/model/metadata/S3BackendMetaData.java | 9 +++++---- .../backend/module/filesystem/s3/utils/S3Utils.java | 4 +++- .../filesystem/sync/FilesystemSyncProcessTest.java | 4 ++++ 10 files changed, 48 insertions(+), 23 deletions(-) diff --git a/pom.xml b/pom.xml index 0c467092..05300b35 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-20220628.161829-14 + 0.0.0-20220629.151616-15 diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index e5c69153..9a04f289 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -81,7 +81,7 @@ public abstract class AbstractBaseFilesystemAction /******************************************************************************* ** Get a string that represents the full path to a file. *******************************************************************************/ - protected abstract String getFullPathForFile(FILE file); + public abstract String getFullPathForFile(FILE file); /******************************************************************************* ** In contrast with the DeleteAction, which deletes RECORDS - this is a @@ -105,7 +105,7 @@ public abstract class AbstractBaseFilesystemAction ** and a file at /foo/bar/baz.txt ** give us just the baz.txt part. *******************************************************************************/ - public abstract String stripBackendAndTableBasePathsFromFileName(FILE file, QBackendMetaData sourceBackend, QTableMetaData sourceTable); + public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 74ddadc2..be5dd44b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -95,7 +95,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** Get a string that represents the full path to a file. *******************************************************************************/ @Override - protected String getFullPathForFile(File file) + public String getFullPathForFile(File file) { return (file.getAbsolutePath()); } @@ -176,10 +176,10 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** give us just the baz.txt part. *******************************************************************************/ @Override - public String stripBackendAndTableBasePathsFromFileName(File file, QBackendMetaData backend, QTableMetaData table) + public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) { String tablePath = getFullBasePath(table, backend); - String strippedPath = file.getAbsolutePath().replaceFirst(".*" + tablePath, ""); + String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); return (strippedPath); } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java index 4e096635..f91a5b10 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -56,7 +57,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete"; public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves"; - public static final String VALUE_MOVE = "move"; + public static final String VALUE_MOVE = "move"; public static final String VALUE_DELETE = "delete"; public static final String FUNCTION_NAME = "cleanupSourceFiles"; @@ -77,6 +78,8 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody { throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); } + AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase(); + actionBase.preAction(backend); String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); if(!StringUtils.hasContent(sourceFilePaths)) @@ -91,7 +94,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE); if(VALUE_DELETE.equals(moveOrDelete)) { - filesystemModule.getActionBase().deleteFile(runFunctionRequest.getInstance(), table, sourceFile); + actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); } else if(VALUE_MOVE.equals(moveOrDelete)) { @@ -100,9 +103,9 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody { throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value.")); } - File file = new File(sourceFile); - String destinationPath = destinationForMoves + File.separator + file.getName(); - filesystemModule.getActionBase().moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); + String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table); + String destinationPath = destinationForMoves + File.separator + filePathWithoutBase; + actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); } else { diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java index 5ca6c732..45264376 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java @@ -82,6 +82,8 @@ public class FilesystemSyncFunction implements FunctionBody AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); processingActionBase.preAction(processingBackend); + Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); + int syncedFileCount = 0; for(Map.Entry sourceEntry : sourceFiles.entrySet()) { try @@ -98,6 +100,13 @@ public class FilesystemSyncFunction implements FunctionBody String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend); processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); + syncedFileCount++; + + if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync) + { + LOG.info("Breaking after syncing " + syncedFileCount + " files"); + break; + } } } catch(Exception e) @@ -119,7 +128,7 @@ public class FilesystemSyncFunction implements FunctionBody for(Object file : files) { - String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(file, backend, table); + String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table); rs.put(fileName, file); } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java index 37313fb2..5773630a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java @@ -41,6 +41,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; ** - if any files exist in the source, but not in the archive, then: ** - copy the file to both the archive and the processing table. ** + ** The maxFilesToArchive field can be used to only sync up to that many files + ** (an help an initial sync, if you want to do it in smaller batches) + ** ** The idea being, that the source is read-only, and we want to move files out of ** processing after they've been processed - and the archive is what we can have ** in-between the two. @@ -49,9 +52,10 @@ public class FilesystemSyncProcess { public static final String PROCESS_NAME = "filesystem.sync"; - public static final String FIELD_SOURCE_TABLE = "sourceTable"; - public static final String FIELD_ARCHIVE_TABLE = "archiveTable"; - public static final String FIELD_PROCESSING_TABLE = "processingTable"; + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_ARCHIVE_TABLE = "archiveTable"; + public static final String FIELD_PROCESSING_TABLE = "processingTable"; + public static final String FIELD_MAX_FILES_TO_ARCHIVE = "maxFilesToArchive"; @@ -69,6 +73,7 @@ public class FilesystemSyncProcess .withInputData(new QFunctionInputMetaData() .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)) .addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING)) + .addField(new QFieldMetaData(FIELD_MAX_FILES_TO_ARCHIVE, QFieldType.INTEGER).withDefaultValue(Integer.MAX_VALUE)) .addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING))); return new QProcessMetaData() diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index a7ae421b..21958ab7 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -175,7 +175,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction