QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-14 fixes to sync process

Browse Source
This commit is contained in:
darin.kelkhoff
2022-06-28 13:49:37 -05:00
parent 1e376b9359
commit 4bc0ab1c49
3 changed files with 40 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -32,7 +32,6 @@ import java.util.function.Function;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -182,7 +181,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/ *******************************************************************************/
public QueryResult executeQuery(QueryRequest queryRequest) throws QException public QueryResult executeQuery(QueryRequest queryRequest) throws QException
{ {
preAction(queryRequest); preAction(queryRequest.getBackend());
try try
{ {
@ -256,7 +255,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
** Method that subclasses can override to add pre-action things (e.g., setting up ** Method that subclasses can override to add pre-action things (e.g., setting up
** s3 client). ** s3 client).
*******************************************************************************/ *******************************************************************************/
protected void preAction(AbstractQTableRequest tableRequest) public void preAction(QBackendMetaData backendMetaData)
{ {
///////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////
// noop in base class - subclasses can add functionality if needed // // noop in base class - subclasses can add functionality if needed //

32
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java
View File

@ -65,16 +65,22 @@ public class FilesystemSyncFunction implements FunctionBody
QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceModule.getActionBase(), sourceTable, sourceBackend); AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveModule.getActionBase(), archiveTable, archiveBackend).keySet(); AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend);
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet()) for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
{ {
@ -84,14 +90,14 @@ public class FilesystemSyncFunction implements FunctionBody
if(!archiveFiles.contains(sourceFileName)) if(!archiveFiles.contains(sourceFileName))
{ {
LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]"); LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
InputStream inputStream = sourceModule.getActionBase().readFile(sourceEntry.getValue()); InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue());
byte[] bytes = inputStream.readAllBytes(); byte[] bytes = inputStream.readAllBytes();
String archivePath = archiveModule.getActionBase().getFullBasePath(archiveTable, archiveBackend); String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
archiveModule.getActionBase().writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
String processingPath = processingModule.getActionBase().getFullBasePath(processingTable, processingBackend); String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
processingModule.getActionBase().writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
} }
} }
catch(Exception e) catch(Exception e)

24
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -29,7 +29,6 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
@ -58,9 +57,9 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
** Setup the s3 utils object to be used for this action. ** Setup the s3 utils object to be used for this action.
*******************************************************************************/ *******************************************************************************/
@Override @Override
protected void preAction(AbstractQTableRequest tableRequest) public void preAction(QBackendMetaData backendMetaData)
{ {
super.preAction(tableRequest); super.preAction(backendMetaData);
if(s3Utils != null) if(s3Utils != null)
{ {
@ -68,7 +67,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
return; return;
} }
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, tableRequest.getBackend()); S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendMetaData);
AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard(); AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
if(StringUtils.hasContent(s3BackendMetaData.getAccessKey()) && StringUtils.hasContent(s3BackendMetaData.getSecretKey())) if(StringUtils.hasContent(s3BackendMetaData.getAccessKey()) && StringUtils.hasContent(s3BackendMetaData.getSecretKey()))
@ -151,12 +150,27 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
@Override @Override
public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
{ {
path = stripDuplicatedSlashes(path); path = stripLeadingSlash(stripDuplicatedSlashes(path));
String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName(); String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
getS3Utils().writeFile(bucketName, path, contents); getS3Utils().writeFile(bucketName, path, contents);
} }
/*******************************************************************************
**
*******************************************************************************/
private String stripLeadingSlash(String path)
{
if(path == null)
{
return (null);
}
return (path.replaceFirst("^/+", ""));
}
/******************************************************************************* /*******************************************************************************
** Get a string that represents the full path to a file. ** Get a string that represents the full path to a file.
*******************************************************************************/ *******************************************************************************/