mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 05:01:07 +00:00
Update for s3 tables, to allow setting content-type in aws when inserting records (files) based on file name, hard-coded value, or another field.
this involved adding table & record params to writeFile method - a @Deprecated wrapper w/o those args is provided for backward compatibility
This commit is contained in:
@ -124,10 +124,19 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public abstract InputStream readFile(FILE file) throws IOException;
|
public abstract InputStream readFile(FILE file) throws IOException;
|
||||||
|
|
||||||
|
/***************************************************************************
|
||||||
|
** Legacy signature for this method - before table & record params were added.
|
||||||
|
***************************************************************************/
|
||||||
|
@Deprecated(since = "call the overload that takes table and record")
|
||||||
|
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
|
||||||
|
{
|
||||||
|
writeFile(backend, null, null, path, contents);
|
||||||
|
}
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
** Write a file - to be implemented in module-specific subclasses.
|
** Write a file - to be implemented in module-specific subclasses.
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException;
|
public abstract void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException;
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
** Get a string that represents the full path to a file.
|
** Get a string that represents the full path to a file.
|
||||||
@ -632,7 +641,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
|
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
|
||||||
writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
|
writeFile(backend, table, record, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
|
||||||
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
|
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
|
||||||
output.addRecord(record);
|
output.addRecord(record);
|
||||||
}
|
}
|
||||||
|
@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
|
|||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||||
@ -204,7 +205,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
|
|||||||
** Write a file - to be implemented in module-specific subclasses.
|
** Write a file - to be implemented in module-specific subclasses.
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
@Override
|
@Override
|
||||||
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
|
public void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
|
||||||
{
|
{
|
||||||
FileUtils.writeByteArrayToFile(new File(path), contents);
|
FileUtils.writeByteArrayToFile(new File(path), contents);
|
||||||
}
|
}
|
||||||
|
@ -363,7 +363,7 @@ public class FilesystemImporterStep implements BackendStep
|
|||||||
path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
|
path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
|
||||||
|
|
||||||
LOG.info("Archiving file", logPair("path", path), logPair("archiveBackendName", archiveBackend.getName()), logPair("archiveTableName", archiveTable.getName()));
|
LOG.info("Archiving file", logPair("path", path), logPair("archiveBackendName", archiveBackend.getName()), logPair("archiveTableName", archiveTable.getName()));
|
||||||
archiveActionBase.writeFile(archiveBackend, path, bytes);
|
archiveActionBase.writeFile(archiveBackend, archiveTable, null, path, bytes);
|
||||||
|
|
||||||
return (path);
|
return (path);
|
||||||
}
|
}
|
||||||
|
@ -111,10 +111,10 @@ public class FilesystemSyncStep implements BackendStep
|
|||||||
byte[] bytes = inputStream.readAllBytes();
|
byte[] bytes = inputStream.readAllBytes();
|
||||||
|
|
||||||
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
|
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
|
||||||
archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
|
archiveActionBase.writeFile(archiveBackend, archiveTable, null, archivePath + File.separator + sourceFileName, bytes);
|
||||||
|
|
||||||
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
|
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
|
||||||
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
|
processingActionBase.writeFile(processingBackend, processingTable, null, processingPath + File.separator + sourceFileName, bytes);
|
||||||
syncedFileCount++;
|
syncedFileCount++;
|
||||||
|
|
||||||
if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
|
if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
|
||||||
|
@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.net.URLConnection;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||||
import com.amazonaws.auth.BasicAWSCredentials;
|
import com.amazonaws.auth.BasicAWSCredentials;
|
||||||
import com.amazonaws.services.s3.AmazonS3;
|
import com.amazonaws.services.s3.AmazonS3;
|
||||||
@ -34,6 +36,7 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
|
|||||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||||
@ -42,6 +45,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile
|
|||||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
|
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
|
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
|
||||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||||
|
|
||||||
@ -195,14 +199,27 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
|
|||||||
** Write a file - to be implemented in module-specific subclasses.
|
** Write a file - to be implemented in module-specific subclasses.
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
@Override
|
@Override
|
||||||
public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
|
public void writeFile(QBackendMetaData backendMetaData, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
|
||||||
{
|
{
|
||||||
String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
|
String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
path = stripLeadingSlash(stripDuplicatedSlashes(path));
|
path = stripLeadingSlash(stripDuplicatedSlashes(path));
|
||||||
getS3Utils().writeFile(bucketName, path, contents);
|
String contentType = null;
|
||||||
|
|
||||||
|
if(table.getBackendDetails() instanceof S3TableBackendDetails s3TableBackendDetails)
|
||||||
|
{
|
||||||
|
contentType = switch(Objects.requireNonNullElse(s3TableBackendDetails.getContentTypeStrategy(), S3TableBackendDetails.ContentTypeStrategy.NONE))
|
||||||
|
{
|
||||||
|
case BASED_ON_FILE_NAME -> URLConnection.guessContentTypeFromName(path);
|
||||||
|
case FROM_FIELD -> record == null ? null : record.getValueString(s3TableBackendDetails.getContentTypeFieldName());
|
||||||
|
case HARDCODED -> s3TableBackendDetails.getHardcodedContentType();
|
||||||
|
case NONE -> null;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
getS3Utils().writeFile(bucketName, path, contents, contentType);
|
||||||
}
|
}
|
||||||
catch(Exception e)
|
catch(Exception e)
|
||||||
{
|
{
|
||||||
@ -277,5 +294,4 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
|
|||||||
getS3Utils().moveObject(bucketName, source, destination);
|
getS3Utils().moveObject(bucketName, source, destination);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -22,6 +22,11 @@
|
|||||||
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
|
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
|
||||||
|
|
||||||
|
|
||||||
|
import java.util.Objects;
|
||||||
|
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
|
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
|
||||||
|
|
||||||
@ -31,6 +36,21 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
|
public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
|
||||||
{
|
{
|
||||||
|
private ContentTypeStrategy contentTypeStrategy = ContentTypeStrategy.NONE;
|
||||||
|
private String contentTypeFieldName;
|
||||||
|
private String hardcodedContentType;
|
||||||
|
|
||||||
|
|
||||||
|
/***************************************************************************
|
||||||
|
**
|
||||||
|
***************************************************************************/
|
||||||
|
public enum ContentTypeStrategy
|
||||||
|
{
|
||||||
|
BASED_ON_FILE_NAME,
|
||||||
|
FROM_FIELD,
|
||||||
|
HARDCODED,
|
||||||
|
NONE
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -43,4 +63,131 @@ public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
|
|||||||
setBackendType(S3BackendModule.class);
|
setBackendType(S3BackendModule.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/***************************************************************************
|
||||||
|
**
|
||||||
|
***************************************************************************/
|
||||||
|
@Override
|
||||||
|
public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
|
||||||
|
{
|
||||||
|
super.validate(qInstance, table, qInstanceValidator);
|
||||||
|
|
||||||
|
String prefix = "Table " + (table == null ? "null" : table.getName()) + " backend details - ";
|
||||||
|
switch (Objects.requireNonNullElse(contentTypeStrategy, ContentTypeStrategy.NONE))
|
||||||
|
{
|
||||||
|
case BASED_ON_FILE_NAME, NONE ->
|
||||||
|
{
|
||||||
|
qInstanceValidator.assertCondition(!StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||||
|
qInstanceValidator.assertCondition(!StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||||
|
}
|
||||||
|
case FROM_FIELD ->
|
||||||
|
{
|
||||||
|
qInstanceValidator.assertCondition(!StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||||
|
|
||||||
|
if(table != null && qInstanceValidator.assertCondition(StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName must be set when contentTypeStrategy is " + contentTypeStrategy))
|
||||||
|
{
|
||||||
|
qInstanceValidator.assertCondition(table.getFields().containsKey(contentTypeFieldName), prefix + "contentTypeFieldName must be a valid field name in the table");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case HARDCODED ->
|
||||||
|
{
|
||||||
|
qInstanceValidator.assertCondition(!StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||||
|
qInstanceValidator.assertCondition(StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType must be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for contentTypeStrategy
|
||||||
|
*******************************************************************************/
|
||||||
|
public ContentTypeStrategy getContentTypeStrategy()
|
||||||
|
{
|
||||||
|
return (this.contentTypeStrategy);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for contentTypeStrategy
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setContentTypeStrategy(ContentTypeStrategy contentTypeStrategy)
|
||||||
|
{
|
||||||
|
this.contentTypeStrategy = contentTypeStrategy;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Fluent setter for contentTypeStrategy
|
||||||
|
*******************************************************************************/
|
||||||
|
public S3TableBackendDetails withContentTypeStrategy(ContentTypeStrategy contentTypeStrategy)
|
||||||
|
{
|
||||||
|
this.contentTypeStrategy = contentTypeStrategy;
|
||||||
|
return (this);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for contentTypeFieldName
|
||||||
|
*******************************************************************************/
|
||||||
|
public String getContentTypeFieldName()
|
||||||
|
{
|
||||||
|
return (this.contentTypeFieldName);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for contentTypeFieldName
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setContentTypeFieldName(String contentTypeFieldName)
|
||||||
|
{
|
||||||
|
this.contentTypeFieldName = contentTypeFieldName;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Fluent setter for contentTypeFieldName
|
||||||
|
*******************************************************************************/
|
||||||
|
public S3TableBackendDetails withContentTypeFieldName(String contentTypeFieldName)
|
||||||
|
{
|
||||||
|
this.contentTypeFieldName = contentTypeFieldName;
|
||||||
|
return (this);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for hardcodedContentType
|
||||||
|
*******************************************************************************/
|
||||||
|
public String getHardcodedContentType()
|
||||||
|
{
|
||||||
|
return (this.hardcodedContentType);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for hardcodedContentType
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setHardcodedContentType(String hardcodedContentType)
|
||||||
|
{
|
||||||
|
this.hardcodedContentType = hardcodedContentType;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Fluent setter for hardcodedContentType
|
||||||
|
*******************************************************************************/
|
||||||
|
public S3TableBackendDetails withHardcodedContentType(String hardcodedContentType)
|
||||||
|
{
|
||||||
|
this.hardcodedContentType = hardcodedContentType;
|
||||||
|
return (this);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -204,10 +204,11 @@ public class S3Utils
|
|||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
** Write a file
|
** Write a file
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public void writeFile(String bucket, String key, byte[] contents)
|
public void writeFile(String bucket, String key, byte[] contents, String contentType)
|
||||||
{
|
{
|
||||||
ObjectMetadata objectMetadata = new ObjectMetadata();
|
ObjectMetadata objectMetadata = new ObjectMetadata();
|
||||||
objectMetadata.setContentLength(contents.length);
|
objectMetadata.setContentLength(contents.length);
|
||||||
|
objectMetadata.setContentType(contentType);
|
||||||
getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), objectMetadata);
|
getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), objectMetadata);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -372,7 +372,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
|
|||||||
**
|
**
|
||||||
***************************************************************************/
|
***************************************************************************/
|
||||||
@Override
|
@Override
|
||||||
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
|
public void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
|
||||||
{
|
{
|
||||||
sftpClient.put(new ByteArrayInputStream(contents), path);
|
sftpClient.put(new ByteArrayInputStream(contents), path);
|
||||||
}
|
}
|
||||||
|
@ -362,6 +362,7 @@ public class TestUtils
|
|||||||
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
|
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
|
||||||
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
|
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
|
||||||
.withBackendDetails(new S3TableBackendDetails()
|
.withBackendDetails(new S3TableBackendDetails()
|
||||||
|
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.BASED_ON_FILE_NAME)
|
||||||
.withBasePath("blobs")
|
.withBasePath("blobs")
|
||||||
.withCardinality(Cardinality.ONE)
|
.withCardinality(Cardinality.ONE)
|
||||||
.withFileNameFieldName("fileName")
|
.withFileNameFieldName("fileName")
|
||||||
|
@ -229,7 +229,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
|
|||||||
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
|
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
|
||||||
String fullPath = actionBase.getFullBasePath(table, backend);
|
String fullPath = actionBase.getFullBasePath(table, backend);
|
||||||
|
|
||||||
actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes());
|
actionBase.writeFile(backend, table, null, fullPath + "/" + name, content.getBytes());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||||
import com.amazonaws.services.s3.model.S3Object;
|
import com.amazonaws.services.s3.model.S3Object;
|
||||||
|
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
|
||||||
@ -34,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
|||||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.commons.lang.NotImplementedException;
|
import org.apache.commons.lang.NotImplementedException;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
@ -54,18 +57,15 @@ public class S3InsertActionTest extends BaseS3Test
|
|||||||
@Test
|
@Test
|
||||||
public void testCardinalityOne() throws QException, IOException
|
public void testCardinalityOne() throws QException, IOException
|
||||||
{
|
{
|
||||||
QInstance qInstance = TestUtils.defineInstance();
|
|
||||||
|
|
||||||
InsertInput insertInput = new InsertInput();
|
InsertInput insertInput = new InsertInput();
|
||||||
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
|
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
|
||||||
insertInput.setRecords(List.of(
|
insertInput.setRecords(List.of(
|
||||||
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
|
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
|
||||||
));
|
|
||||||
|
|
||||||
S3InsertAction insertAction = new S3InsertAction();
|
S3InsertAction insertAction = new S3InsertAction();
|
||||||
insertAction.setS3Utils(getS3Utils());
|
insertAction.setS3Utils(getS3Utils());
|
||||||
|
|
||||||
InsertOutput insertOutput = insertAction.execute(insertInput);
|
InsertOutput insertOutput = insertAction.execute(insertInput);
|
||||||
|
|
||||||
assertThat(insertOutput.getRecords())
|
assertThat(insertOutput.getRecords())
|
||||||
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
|
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
|
||||||
|
|
||||||
@ -73,6 +73,65 @@ public class S3InsertActionTest extends BaseS3Test
|
|||||||
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
|
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
|
||||||
List<String> lines = IOUtils.readLines(object.getObjectContent(), StandardCharsets.UTF_8);
|
List<String> lines = IOUtils.readLines(object.getObjectContent(), StandardCharsets.UTF_8);
|
||||||
assertEquals("Hi, Bob.", lines.get(0));
|
assertEquals("Hi, Bob.", lines.get(0));
|
||||||
|
|
||||||
|
ObjectMetadata objectMetadata = object.getObjectMetadata();
|
||||||
|
assertEquals("text/plain", objectMetadata.getContentType());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testContentTypeFromField() throws QException
|
||||||
|
{
|
||||||
|
((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
|
||||||
|
.getBackendDetails())
|
||||||
|
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.FROM_FIELD)
|
||||||
|
.withContentTypeFieldName("contentType");
|
||||||
|
|
||||||
|
InsertInput insertInput = new InsertInput();
|
||||||
|
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
|
||||||
|
insertInput.setRecords(List.of(
|
||||||
|
new QRecord().withValue("fileName", "file2.txt").withValue("contentType", "myContentType/fake").withValue("contents", "Hi, Bob.")));
|
||||||
|
|
||||||
|
S3InsertAction insertAction = new S3InsertAction();
|
||||||
|
insertAction.setS3Utils(getS3Utils());
|
||||||
|
InsertOutput insertOutput = insertAction.execute(insertInput);
|
||||||
|
|
||||||
|
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
|
||||||
|
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
|
||||||
|
ObjectMetadata objectMetadata = object.getObjectMetadata();
|
||||||
|
assertEquals("myContentType/fake", objectMetadata.getContentType());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testContentTypeHardcoded() throws QException
|
||||||
|
{
|
||||||
|
((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
|
||||||
|
.getBackendDetails())
|
||||||
|
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.HARDCODED)
|
||||||
|
.withHardcodedContentType("your-content-type");
|
||||||
|
|
||||||
|
InsertInput insertInput = new InsertInput();
|
||||||
|
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
|
||||||
|
insertInput.setRecords(List.of(
|
||||||
|
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
|
||||||
|
|
||||||
|
S3InsertAction insertAction = new S3InsertAction();
|
||||||
|
insertAction.setS3Utils(getS3Utils());
|
||||||
|
InsertOutput insertOutput = insertAction.execute(insertInput);
|
||||||
|
|
||||||
|
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
|
||||||
|
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
|
||||||
|
ObjectMetadata objectMetadata = object.getObjectMetadata();
|
||||||
|
assertEquals("your-content-type", objectMetadata.getContentType());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -0,0 +1,177 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
|
||||||
|
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||||
|
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||||
|
import org.assertj.core.api.CollectionAssert;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Unit test for S3TableBackendDetails
|
||||||
|
*******************************************************************************/
|
||||||
|
class S3TableBackendDetailsTest extends BaseTest
|
||||||
|
{
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testValidateContentTypeStrategyBasedOnFileNameOrNone()
|
||||||
|
{
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
// same validation rules for both of these //
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
for(S3TableBackendDetails.ContentTypeStrategy contentTypeStrategy : ListBuilder.of(null, S3TableBackendDetails.ContentTypeStrategy.BASED_ON_FILE_NAME, S3TableBackendDetails.ContentTypeStrategy.NONE))
|
||||||
|
{
|
||||||
|
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
|
||||||
|
.withContentTypeStrategy(contentTypeStrategy);
|
||||||
|
QTableMetaData table = getQTableMetaData();
|
||||||
|
|
||||||
|
List<String> errors = runValidation(s3TableBackendDetails, table);
|
||||||
|
CollectionAssert.assertThatCollection(errors)
|
||||||
|
.isEmpty();
|
||||||
|
|
||||||
|
s3TableBackendDetails.setHardcodedContentType("Test");
|
||||||
|
s3TableBackendDetails.setContentTypeFieldName("Test");
|
||||||
|
errors = runValidation(s3TableBackendDetails, table);
|
||||||
|
CollectionAssert.assertThatCollection(errors)
|
||||||
|
.hasSize(2)
|
||||||
|
.contains("Table testTable backend details - contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy)
|
||||||
|
.contains("Table testTable backend details - hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testValidateContentTypeStrategyFromField()
|
||||||
|
{
|
||||||
|
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
|
||||||
|
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.FROM_FIELD);
|
||||||
|
QTableMetaData table = getQTableMetaData();
|
||||||
|
|
||||||
|
List<String> errors = runValidation(s3TableBackendDetails, table);
|
||||||
|
CollectionAssert.assertThatCollection(errors)
|
||||||
|
.hasSize(1)
|
||||||
|
.contains("Table testTable backend details - contentTypeFieldName must be set when contentTypeStrategy is FROM_FIELD");
|
||||||
|
|
||||||
|
s3TableBackendDetails.setContentTypeFieldName("notAField");
|
||||||
|
errors = runValidation(s3TableBackendDetails, table);
|
||||||
|
CollectionAssert.assertThatCollection(errors)
|
||||||
|
.hasSize(1)
|
||||||
|
.contains("Table testTable backend details - contentTypeFieldName must be a valid field name in the table");
|
||||||
|
|
||||||
|
table.addField(new QFieldMetaData("contentType", QFieldType.STRING));
|
||||||
|
s3TableBackendDetails.setContentTypeFieldName("contentType");
|
||||||
|
errors = runValidation(s3TableBackendDetails, table);
|
||||||
|
CollectionAssert.assertThatCollection(errors)
|
||||||
|
.isEmpty();
|
||||||
|
|
||||||
|
s3TableBackendDetails.setHardcodedContentType("hard");
|
||||||
|
errors = runValidation(s3TableBackendDetails, table);
|
||||||
|
CollectionAssert.assertThatCollection(errors)
|
||||||
|
.hasSize(1)
|
||||||
|
.contains("Table testTable backend details - hardcodedContentType should not be set when contentTypeStrategy is FROM_FIELD");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testValidateContentTypeStrategyHardcoded()
|
||||||
|
{
|
||||||
|
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
|
||||||
|
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.HARDCODED);
|
||||||
|
QTableMetaData table = getQTableMetaData();
|
||||||
|
|
||||||
|
List<String> errors = runValidation(s3TableBackendDetails, table);
|
||||||
|
CollectionAssert.assertThatCollection(errors)
|
||||||
|
.hasSize(1)
|
||||||
|
.contains("Table testTable backend details - hardcodedContentType must be set when contentTypeStrategy is HARDCODED");
|
||||||
|
|
||||||
|
s3TableBackendDetails.setHardcodedContentType("Test");
|
||||||
|
errors = runValidation(s3TableBackendDetails, table);
|
||||||
|
CollectionAssert.assertThatCollection(errors)
|
||||||
|
.isEmpty();
|
||||||
|
|
||||||
|
s3TableBackendDetails.setContentTypeFieldName("aField");
|
||||||
|
errors = runValidation(s3TableBackendDetails, table);
|
||||||
|
CollectionAssert.assertThatCollection(errors)
|
||||||
|
.hasSize(1)
|
||||||
|
.contains("Table testTable backend details - contentTypeFieldName should not be set when contentTypeStrategy is HARDCODED");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/***************************************************************************
|
||||||
|
**
|
||||||
|
***************************************************************************/
|
||||||
|
private static QTableMetaData getQTableMetaData()
|
||||||
|
{
|
||||||
|
QTableMetaData table = new QTableMetaData()
|
||||||
|
.withName("testTable")
|
||||||
|
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
|
||||||
|
.withField(new QFieldMetaData("fileName", QFieldType.STRING));
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/***************************************************************************
|
||||||
|
**
|
||||||
|
***************************************************************************/
|
||||||
|
private static S3TableBackendDetails getS3TableBackendDetails()
|
||||||
|
{
|
||||||
|
S3TableBackendDetails s3TableBackendDetails = new S3TableBackendDetails()
|
||||||
|
.withContentsFieldName("contents")
|
||||||
|
.withFileNameFieldName("fileName")
|
||||||
|
.withCardinality(Cardinality.ONE);
|
||||||
|
return s3TableBackendDetails;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/***************************************************************************
|
||||||
|
**
|
||||||
|
***************************************************************************/
|
||||||
|
private List<String> runValidation(S3TableBackendDetails s3TableBackendDetails, QTableMetaData table)
|
||||||
|
{
|
||||||
|
QInstanceValidator validator = new QInstanceValidator();
|
||||||
|
s3TableBackendDetails.validate(QContext.getQInstance(), table, validator);
|
||||||
|
return (validator.getErrors());
|
||||||
|
}
|
||||||
|
}
|
Reference in New Issue
Block a user