QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-14 fixes to sync process

Browse Source
This commit is contained in:
darin.kelkhoff
2022-06-28 13:49:37 -05:00
parent 1e376b9359
commit 4bc0ab1c49
3 changed files with 40 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -32,7 +32,6 @@ import java.util.function.Function;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -182,7 +181,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/
public QueryResult executeQuery(QueryRequest queryRequest) throws QException
{
preAction(queryRequest);
preAction(queryRequest.getBackend());
try
{
@ -256,7 +255,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
** Method that subclasses can override to add pre-action things (e.g., setting up
** s3 client).
*******************************************************************************/
protected void preAction(AbstractQTableRequest tableRequest)
public void preAction(QBackendMetaData backendMetaData)
{
/////////////////////////////////////////////////////////////////////
// noop in base class - subclasses can add functionality if needed //

20
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java
View File

@ -67,14 +67,20 @@ public class FilesystemSyncFunction implements FunctionBody
QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceModule.getActionBase(), sourceTable, sourceBackend);
AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveModule.getActionBase(), archiveTable, archiveBackend).keySet();
AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend);
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
{
@ -84,14 +90,14 @@ public class FilesystemSyncFunction implements FunctionBody
if(!archiveFiles.contains(sourceFileName))
{
LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
InputStream inputStream = sourceModule.getActionBase().readFile(sourceEntry.getValue());
InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue());
byte[] bytes = inputStream.readAllBytes();
String archivePath = archiveModule.getActionBase().getFullBasePath(archiveTable, archiveBackend);
archiveModule.getActionBase().writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
String processingPath = processingModule.getActionBase().getFullBasePath(processingTable, processingBackend);
processingModule.getActionBase().writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
}
}
catch(Exception e)

24
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -29,7 +29,6 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
@ -58,9 +57,9 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
** Setup the s3 utils object to be used for this action.
*******************************************************************************/
@Override
protected void preAction(AbstractQTableRequest tableRequest)
public void preAction(QBackendMetaData backendMetaData)
{
super.preAction(tableRequest);
super.preAction(backendMetaData);
if(s3Utils != null)
{
@ -68,7 +67,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
return;
}
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, tableRequest.getBackend());
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendMetaData);
AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
if(StringUtils.hasContent(s3BackendMetaData.getAccessKey()) && StringUtils.hasContent(s3BackendMetaData.getSecretKey()))
@ -151,12 +150,27 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
@Override
public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
{
path = stripDuplicatedSlashes(path);
path = stripLeadingSlash(stripDuplicatedSlashes(path));
String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
getS3Utils().writeFile(bucketName, path, contents);
}
/*******************************************************************************
**
*******************************************************************************/
private String stripLeadingSlash(String path)
{
if(path == null)
{
return (null);
}
return (path.replaceFirst("^/+", ""));
}
/*******************************************************************************
** Get a string that represents the full path to a file.
*******************************************************************************/