diff --git a/pom.xml b/pom.xml
index 0c467092..05300b35 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,7 +51,7 @@
com.kingsrook.qqq
qqq-backend-core
- 0.0.0-20220628.161829-14
+ 0.0.0-20220629.151616-15
diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
index e5c69153..9a04f289 100644
--- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
+++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
@@ -81,7 +81,7 @@ public abstract class AbstractBaseFilesystemAction
/*******************************************************************************
** Get a string that represents the full path to a file.
*******************************************************************************/
- protected abstract String getFullPathForFile(FILE file);
+ public abstract String getFullPathForFile(FILE file);
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
@@ -105,7 +105,7 @@ public abstract class AbstractBaseFilesystemAction
** and a file at /foo/bar/baz.txt
** give us just the baz.txt part.
*******************************************************************************/
- public abstract String stripBackendAndTableBasePathsFromFileName(FILE file, QBackendMetaData sourceBackend, QTableMetaData sourceTable);
+ public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable);
diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
index 74ddadc2..be5dd44b 100644
--- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
+++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
@@ -95,7 +95,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction
** Get a string that represents the full path to a file.
*******************************************************************************/
@Override
- protected String getFullPathForFile(File file)
+ public String getFullPathForFile(File file)
{
return (file.getAbsolutePath());
}
@@ -176,10 +176,10 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction
** give us just the baz.txt part.
*******************************************************************************/
@Override
- public String stripBackendAndTableBasePathsFromFileName(File file, QBackendMetaData backend, QTableMetaData table)
+ public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table)
{
String tablePath = getFullBasePath(table, backend);
- String strippedPath = file.getAbsolutePath().replaceFirst(".*" + tablePath, "");
+ String strippedPath = filePath.replaceFirst(".*" + tablePath, "");
return (strippedPath);
}
diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java
index 4e096635..f91a5b10 100644
--- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java
+++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java
@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -56,7 +57,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete";
public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves";
- public static final String VALUE_MOVE = "move";
+ public static final String VALUE_MOVE = "move";
public static final String VALUE_DELETE = "delete";
public static final String FUNCTION_NAME = "cleanupSourceFiles";
@@ -77,6 +78,8 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
{
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
}
+ AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase();
+ actionBase.preAction(backend);
String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
if(!StringUtils.hasContent(sourceFilePaths))
@@ -91,7 +94,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE);
if(VALUE_DELETE.equals(moveOrDelete))
{
- filesystemModule.getActionBase().deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
+ actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
}
else if(VALUE_MOVE.equals(moveOrDelete))
{
@@ -100,9 +103,9 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
{
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
}
- File file = new File(sourceFile);
- String destinationPath = destinationForMoves + File.separator + file.getName();
- filesystemModule.getActionBase().moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
+ String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table);
+ String destinationPath = destinationForMoves + File.separator + filePathWithoutBase;
+ actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
}
else
{
diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java
index 5ca6c732..45264376 100644
--- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java
+++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java
@@ -82,6 +82,8 @@ public class FilesystemSyncFunction implements FunctionBody
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend);
+ Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
+ int syncedFileCount = 0;
for(Map.Entry sourceEntry : sourceFiles.entrySet())
{
try
@@ -98,6 +100,13 @@ public class FilesystemSyncFunction implements FunctionBody
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
+ syncedFileCount++;
+
+ if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
+ {
+ LOG.info("Breaking after syncing " + syncedFileCount + " files");
+ break;
+ }
}
}
catch(Exception e)
@@ -119,7 +128,7 @@ public class FilesystemSyncFunction implements FunctionBody
for(Object file : files)
{
- String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(file, backend, table);
+ String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
rs.put(fileName, file);
}
diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java
index 37313fb2..5773630a 100644
--- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java
+++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java
@@ -41,6 +41,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
** - if any files exist in the source, but not in the archive, then:
** - copy the file to both the archive and the processing table.
**
+ ** The maxFilesToArchive field can be used to only sync up to that many files
+ ** (an help an initial sync, if you want to do it in smaller batches)
+ **
** The idea being, that the source is read-only, and we want to move files out of
** processing after they've been processed - and the archive is what we can have
** in-between the two.
@@ -49,9 +52,10 @@ public class FilesystemSyncProcess
{
public static final String PROCESS_NAME = "filesystem.sync";
- public static final String FIELD_SOURCE_TABLE = "sourceTable";
- public static final String FIELD_ARCHIVE_TABLE = "archiveTable";
- public static final String FIELD_PROCESSING_TABLE = "processingTable";
+ public static final String FIELD_SOURCE_TABLE = "sourceTable";
+ public static final String FIELD_ARCHIVE_TABLE = "archiveTable";
+ public static final String FIELD_PROCESSING_TABLE = "processingTable";
+ public static final String FIELD_MAX_FILES_TO_ARCHIVE = "maxFilesToArchive";
@@ -69,6 +73,7 @@ public class FilesystemSyncProcess
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING))
+ .addField(new QFieldMetaData(FIELD_MAX_FILES_TO_ARCHIVE, QFieldType.INTEGER).withDefaultValue(Integer.MAX_VALUE))
.addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING)));
return new QProcessMetaData()
diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
index a7ae421b..21958ab7 100644
--- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
+++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
@@ -175,7 +175,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction