QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update for s3 tables, to allow setting content-type in aws when inserting records (files) based on file name, hard-coded value, or another field.

Browse Source 
this involved adding table & record params to writeFile method - a @Deprecated wrapper w/o those args is provided for backward compatibility
This commit is contained in:
darin.kelkhoff
2025-04-01 15:50:16 -05:00
parent 96c539b323
commit 94631585ee
12 changed files with 428 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -362,6 +362,7 @@ public class TestUtils
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new S3TableBackendDetails()
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.BASED_ON_FILE_NAME)
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")

2
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java
View File

@ -229,7 +229,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
String fullPath = actionBase.getFullBasePath(table, backend);
actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes());
actionBase.writeFile(backend, table, null, fullPath + "/" + name, content.getBytes());
}

69
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertActionTest.java
View File

@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@ -34,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
@ -54,18 +57,15 @@ public class S3InsertActionTest extends BaseS3Test
@Test
public void testCardinalityOne() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
S3InsertAction insertAction = new S3InsertAction();
insertAction.setS3Utils(getS3Utils());
InsertOutput insertOutput = insertAction.execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
@ -73,6 +73,65 @@ public class S3InsertActionTest extends BaseS3Test
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
List<String> lines = IOUtils.readLines(object.getObjectContent(), StandardCharsets.UTF_8);
assertEquals("Hi, Bob.", lines.get(0));
ObjectMetadata objectMetadata = object.getObjectMetadata();
assertEquals("text/plain", objectMetadata.getContentType());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testContentTypeFromField() throws QException
{
((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
.getBackendDetails())
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.FROM_FIELD)
.withContentTypeFieldName("contentType");
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contentType", "myContentType/fake").withValue("contents", "Hi, Bob.")));
S3InsertAction insertAction = new S3InsertAction();
insertAction.setS3Utils(getS3Utils());
InsertOutput insertOutput = insertAction.execute(insertInput);
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
ObjectMetadata objectMetadata = object.getObjectMetadata();
assertEquals("myContentType/fake", objectMetadata.getContentType());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testContentTypeHardcoded() throws QException
{
((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
.getBackendDetails())
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.HARDCODED)
.withHardcodedContentType("your-content-type");
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
S3InsertAction insertAction = new S3InsertAction();
insertAction.setS3Utils(getS3Utils());
InsertOutput insertOutput = insertAction.execute(insertInput);
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
ObjectMetadata objectMetadata = object.getObjectMetadata();
assertEquals("your-content-type", objectMetadata.getContentType());
}

177
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetailsTest.java Normal file
View File

@ -0,0 +1,177 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import java.util.List;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import org.assertj.core.api.CollectionAssert;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for S3TableBackendDetails
*******************************************************************************/
class S3TableBackendDetailsTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testValidateContentTypeStrategyBasedOnFileNameOrNone()
{
/////////////////////////////////////////////
// same validation rules for both of these //
/////////////////////////////////////////////
for(S3TableBackendDetails.ContentTypeStrategy contentTypeStrategy : ListBuilder.of(null, S3TableBackendDetails.ContentTypeStrategy.BASED_ON_FILE_NAME, S3TableBackendDetails.ContentTypeStrategy.NONE))
{
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
.withContentTypeStrategy(contentTypeStrategy);
QTableMetaData table = getQTableMetaData();
List<String> errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.isEmpty();
s3TableBackendDetails.setHardcodedContentType("Test");
s3TableBackendDetails.setContentTypeFieldName("Test");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(2)
.contains("Table testTable backend details - contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy)
.contains("Table testTable backend details - hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testValidateContentTypeStrategyFromField()
{
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.FROM_FIELD);
QTableMetaData table = getQTableMetaData();
List<String> errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - contentTypeFieldName must be set when contentTypeStrategy is FROM_FIELD");
s3TableBackendDetails.setContentTypeFieldName("notAField");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - contentTypeFieldName must be a valid field name in the table");
table.addField(new QFieldMetaData("contentType", QFieldType.STRING));
s3TableBackendDetails.setContentTypeFieldName("contentType");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.isEmpty();
s3TableBackendDetails.setHardcodedContentType("hard");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - hardcodedContentType should not be set when contentTypeStrategy is FROM_FIELD");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testValidateContentTypeStrategyHardcoded()
{
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.HARDCODED);
QTableMetaData table = getQTableMetaData();
List<String> errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - hardcodedContentType must be set when contentTypeStrategy is HARDCODED");
s3TableBackendDetails.setHardcodedContentType("Test");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.isEmpty();
s3TableBackendDetails.setContentTypeFieldName("aField");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - contentTypeFieldName should not be set when contentTypeStrategy is HARDCODED");
}
/***************************************************************************
**
***************************************************************************/
private static QTableMetaData getQTableMetaData()
{
QTableMetaData table = new QTableMetaData()
.withName("testTable")
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withField(new QFieldMetaData("fileName", QFieldType.STRING));
return table;
}
/***************************************************************************
**
***************************************************************************/
private static S3TableBackendDetails getS3TableBackendDetails()
{
S3TableBackendDetails s3TableBackendDetails = new S3TableBackendDetails()
.withContentsFieldName("contents")
.withFileNameFieldName("fileName")
.withCardinality(Cardinality.ONE);
return s3TableBackendDetails;
}
/***************************************************************************
**
***************************************************************************/
private List<String> runValidation(S3TableBackendDetails s3TableBackendDetails, QTableMetaData table)
{
QInstanceValidator validator = new QInstanceValidator();
s3TableBackendDetails.validate(QContext.getQInstance(), table, validator);
return (validator.getErrors());
}
}