Compare commits

...

18 Commits

Author SHA1 Message Date
1599313b75 Bump org.apache.poi:poi-ooxml
Bumps the maven group with 1 update in the /qqq-backend-core directory: org.apache.poi:poi-ooxml.


Updates `org.apache.poi:poi-ooxml` from 5.2.5 to 5.4.0

---
updated-dependencies:
- dependency-name: org.apache.poi:poi-ooxml
  dependency-version: 5.4.0
  dependency-type: direct:production
  dependency-group: maven
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-04-10 15:23:37 +00:00
a4ffe815b5 Merged feature/filesystem-list-single-file-optimization into dev 2025-04-09 11:22:14 -05:00
3f75add3ed added non-ascii to ascii library, timer pretty print 2025-04-08 18:01:43 -05:00
6f1e9413f6 Update for use-case of Get - listing a single file - to pass that file name in, to avoid listing huge directory when not needed 2025-04-08 13:35:08 -05:00
64278e674b Merged feature/dk-misc-20250327 into dev 2025-04-03 14:24:52 -05:00
2fa829658f Merged feature/s3-table-set-content-type-on-insert into dev 2025-04-03 14:24:37 -05:00
8f751d81fe Merged feature/fix-s3-glob-pattern-bad-chars into dev 2025-04-03 14:24:27 -05:00
d42b67582a Merged feature/api-request-updates into dev 2025-04-03 14:24:06 -05:00
942134b4b0 it didn't like default as part of a case, so, moved 2025-04-01 16:52:35 -05:00
aca8436c56 Checkstyle 2025-04-01 16:45:25 -05:00
94631585ee Update for s3 tables, to allow setting content-type in aws when inserting records (files) based on file name, hard-coded value, or another field.
this involved adding table & record params to writeFile method - a @Deprecated wrapper w/o those args is provided for backward compatibility
2025-04-01 15:50:16 -05:00
96c539b323 Update content field to be 12 grid columns [skip ci] 2025-04-01 11:51:48 -05:00
235cf9e16c Bugfix for s3 utils listObjectsInBucketMatchingGlob, for file names with chars that need URL Encoding (since we're using a pathMatcher class and file:/// URIs...) update test setup to have a file that triggered this error before the fix. 2025-04-01 11:09:35 -05:00
d733ce9566 Merged dev into feature/dk-misc-20250327 2025-03-27 12:08:00 -05:00
ebd9dc9c2c Add methods to work with associated records from the mainRecord 2025-03-27 11:57:37 -05:00
12e194fc2e Update all getValueXYZ methods to go through getValue method, so that subclasses behave more as expected 2025-03-27 11:57:09 -05:00
55d046cd86 Fix handling of defaultValue() in annotation 2025-03-27 11:56:00 -05:00
16cedfeb6e Update ConvertHtmlToPdfAction to use openhtmltopdf instead of flying-saucer-pdf-openpdf (gaining support for min/max-width/height 2025-03-27 11:55:36 -05:00
21 changed files with 672 additions and 72 deletions

View File

@ -65,7 +65,11 @@
<artifactId>aws-java-sdk-secretsmanager</artifactId>
<version>1.12.385</version>
</dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
<version>77.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
@ -115,7 +119,7 @@
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>5.2.5</version>
<version>5.4.0</version>
</dependency>
<!-- adding to help FastExcel -->
@ -151,16 +155,21 @@
<version>2.3</version>
</dependency>
<!-- the next 2 deps are for html to pdf - per https://www.baeldung.com/java-html-to-pdf -->
<!-- the next 3 deps are for html to pdf -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.15.3</version>
</dependency>
<dependency>
<groupId>org.xhtmlrenderer</groupId>
<artifactId>flying-saucer-pdf-openpdf</artifactId>
<version>9.1.22</version>
<groupId>com.openhtmltopdf</groupId>
<artifactId>openhtmltopdf-core</artifactId>
<version>1.0.10</version>
</dependency>
<dependency>
<groupId>com.openhtmltopdf</groupId>
<artifactId>openhtmltopdf-pdfbox</artifactId>
<version>1.0.10</version>
</dependency>
<!-- the next 3 deps are being added for google drive support -->

View File

@ -26,22 +26,31 @@ import java.nio.file.Path;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.templates.ConvertHtmlToPdfInput;
import com.kingsrook.qqq.backend.core.model.actions.templates.ConvertHtmlToPdfOutput;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.openhtmltopdf.css.constants.IdentValue;
import com.openhtmltopdf.pdfboxout.PdfBoxFontResolver;
import com.openhtmltopdf.pdfboxout.PdfBoxRenderer;
import com.openhtmltopdf.pdfboxout.PdfRendererBuilder;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.xhtmlrenderer.layout.SharedContext;
import org.xhtmlrenderer.pdf.ITextRenderer;
/*******************************************************************************
** Action to convert a string of HTML to a PDF!
**
** Much credit to https://www.baeldung.com/java-html-to-pdf
*******************************************************************************/
**
** Updated in March 2025 to go from flying-saucer-pdf-openpdf lib to openhtmltopdf,
** mostly to get support for max-height on images...
********************************************************************************/
public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlToPdfInput, ConvertHtmlToPdfOutput>
{
private static final QLogger LOG = QLogger.getLogger(ConvertHtmlToPdfAction.class);
/*******************************************************************************
**
@ -62,31 +71,32 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
//////////////////////////////
// convert the XHTML to PDF //
//////////////////////////////
ITextRenderer renderer = new ITextRenderer();
SharedContext sharedContext = renderer.getSharedContext();
sharedContext.setPrint(true);
sharedContext.setInteractive(false);
PdfRendererBuilder builder = new PdfRendererBuilder();
builder.toStream(input.getOutputStream());
builder.useFastMode();
builder.withHtmlContent(document.html(), input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
if(input.getBasePath() != null)
try(PdfBoxRenderer pdfBoxRenderer = builder.buildPdfRenderer())
{
String baseUrl = input.getBasePath().toUri().toURL().toString();
renderer.setDocumentFromString(document.html(), baseUrl);
}
else
{
renderer.setDocumentFromString(document.html());
}
pdfBoxRenderer.layout();
pdfBoxRenderer.getSharedContext().setPrint(true);
pdfBoxRenderer.getSharedContext().setInteractive(false);
//////////////////////////////////////////////////
// register any custom fonts the input supplied //
//////////////////////////////////////////////////
for(Map.Entry<String, Path> entry : CollectionUtils.nonNullMap(input.getCustomFonts()).entrySet())
{
renderer.getFontResolver().addFont(entry.getValue().toAbsolutePath().toString(), entry.getKey(), "UTF-8", true, null);
}
for(Map.Entry<String, Path> entry : CollectionUtils.nonNullMap(input.getCustomFonts()).entrySet())
{
LOG.warn("Note: Custom fonts appear to not be working in this class at this time...");
pdfBoxRenderer.getFontResolver().addFont(
entry.getValue().toAbsolutePath().toFile(), // Path to the TrueType font file
entry.getKey(), // Font family name to use in CSS
400, // Font weight (e.g., 400 for normal, 700 for bold)
IdentValue.NORMAL, // Font style (e.g., NORMAL, ITALIC)
true, // Whether to subset the font
PdfBoxFontResolver.FontGroup.MAIN // ??
);
}
renderer.layout();
renderer.createPDF(input.getOutputStream());
pdfBoxRenderer.createPDF();
}
return (output);
}

View File

@ -468,7 +468,7 @@ public class QRecord implements Serializable
*******************************************************************************/
public String getValueString(String fieldName)
{
return (ValueUtils.getValueAsString(values.get(fieldName)));
return (ValueUtils.getValueAsString(getValue(fieldName)));
}
@ -479,7 +479,7 @@ public class QRecord implements Serializable
*******************************************************************************/
public Integer getValueInteger(String fieldName)
{
return (ValueUtils.getValueAsInteger(values.get(fieldName)));
return (ValueUtils.getValueAsInteger(getValue(fieldName)));
}
@ -490,7 +490,7 @@ public class QRecord implements Serializable
*******************************************************************************/
public Long getValueLong(String fieldName)
{
return (ValueUtils.getValueAsLong(values.get(fieldName)));
return (ValueUtils.getValueAsLong(getValue(fieldName)));
}
@ -500,7 +500,7 @@ public class QRecord implements Serializable
*******************************************************************************/
public BigDecimal getValueBigDecimal(String fieldName)
{
return (ValueUtils.getValueAsBigDecimal(values.get(fieldName)));
return (ValueUtils.getValueAsBigDecimal(getValue(fieldName)));
}
@ -510,7 +510,7 @@ public class QRecord implements Serializable
*******************************************************************************/
public Boolean getValueBoolean(String fieldName)
{
return (ValueUtils.getValueAsBoolean(values.get(fieldName)));
return (ValueUtils.getValueAsBoolean(getValue(fieldName)));
}
@ -520,7 +520,7 @@ public class QRecord implements Serializable
*******************************************************************************/
public LocalTime getValueLocalTime(String fieldName)
{
return (ValueUtils.getValueAsLocalTime(values.get(fieldName)));
return (ValueUtils.getValueAsLocalTime(getValue(fieldName)));
}
@ -530,7 +530,7 @@ public class QRecord implements Serializable
*******************************************************************************/
public LocalDate getValueLocalDate(String fieldName)
{
return (ValueUtils.getValueAsLocalDate(values.get(fieldName)));
return (ValueUtils.getValueAsLocalDate(getValue(fieldName)));
}
@ -540,7 +540,7 @@ public class QRecord implements Serializable
*******************************************************************************/
public byte[] getValueByteArray(String fieldName)
{
return (ValueUtils.getValueAsByteArray(values.get(fieldName)));
return (ValueUtils.getValueAsByteArray(getValue(fieldName)));
}
@ -550,7 +550,7 @@ public class QRecord implements Serializable
*******************************************************************************/
public Instant getValueInstant(String fieldName)
{
return (ValueUtils.getValueAsInstant(values.get(fieldName)));
return (ValueUtils.getValueAsInstant(getValue(fieldName)));
}

View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.data;
import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
@ -198,4 +199,62 @@ public class QRecordWithJoinedRecords extends QRecord
return (rs);
}
/***************************************************************************
**
***************************************************************************/
@Override
public Map<String, List<QRecord>> getAssociatedRecords()
{
return mainRecord.getAssociatedRecords();
}
/***************************************************************************
**
***************************************************************************/
@Override
public QRecord withAssociatedRecord(String name, QRecord associatedRecord)
{
mainRecord.withAssociatedRecord(name, associatedRecord);
return (this);
}
/***************************************************************************
**
***************************************************************************/
@Override
public QRecord withAssociatedRecords(Map<String, List<QRecord>> associatedRecords)
{
mainRecord.withAssociatedRecords(associatedRecords);
return (this);
}
/***************************************************************************
**
***************************************************************************/
@Override
public void setAssociatedRecords(Map<String, List<QRecord>> associatedRecords)
{
mainRecord.setAssociatedRecords(associatedRecords);
}
/***************************************************************************
**
***************************************************************************/
@Override
public QRecord withAssociatedRecords(String name, List<QRecord> associatedRecords)
{
mainRecord.withAssociatedRecords(name, associatedRecords);
return (this);
}
}

View File

@ -82,6 +82,7 @@ public class HelpContentMetaDataProvider
table.getField("key").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
table.getField("content").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
table.getField("content").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("html")));
table.getField("content").withGridColumns(12);
if(backendDetailEnricher != null)
{

View File

@ -26,6 +26,7 @@ import java.util.Collection;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.ibm.icu.text.Transliterator;
/*******************************************************************************
@ -462,6 +463,17 @@ public class StringUtils
/***************************************************************************
**
***************************************************************************/
public static String replaceNonAsciiCharacters(String s)
{
Transliterator transliterator = Transliterator.getInstance("Any-Latin; Latin-ASCII");
return (transliterator.transliterate(s));
}
/***************************************************************************
**
***************************************************************************/

View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.utils;
import java.time.Duration;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import org.apache.logging.log4j.Level;
@ -79,9 +80,36 @@ public class Timer
**
*******************************************************************************/
public void mark(String message)
{
mark(message, false);
}
/*******************************************************************************
**
*******************************************************************************/
public void mark(String message, boolean prettyPrint)
{
long now = System.currentTimeMillis();
LOG.log(level, String.format("%s: Last [%5d] Total [%5d] %s", name, (now - last), (now - start), message));
if(!prettyPrint)
{
LOG.log(level, String.format("%s: Last [%5d] Total [%5d] %s", name, (now - last), (now - start), message));
}
else
{
Duration lastDuration = Duration.ofMillis(now - last);
Duration totalDuration = Duration.ofMillis(now - start);
LOG.log(level, String.format(
"%s: Last [%d hours, %d minutes, %d seconds, %d milliseconds] Total [%d hours, %d minutes, %d seconds, %d milliseconds] %s",
name, lastDuration.toHours(), lastDuration.toMinutesPart(), lastDuration.toSecondsPart(), lastDuration.toMillisPart(),
totalDuration.toHours(), totalDuration.toMinutesPart(), totalDuration.toSecondsPart(), totalDuration.toMillisPart(),
message));
}
last = now;
}
}

View File

@ -59,9 +59,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSett
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
@ -124,10 +126,19 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/
public abstract InputStream readFile(FILE file) throws IOException;
/***************************************************************************
** Legacy signature for this method - before table & record params were added.
***************************************************************************/
@Deprecated(since = "call the overload that takes table and record")
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
{
writeFile(backend, null, null, path, contents);
}
/*******************************************************************************
** Write a file - to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException;
public abstract void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException;
/*******************************************************************************
** Get a string that represents the full path to a file.
@ -287,8 +298,23 @@ public abstract class AbstractBaseFilesystemAction<FILE>
QueryOutput queryOutput = new QueryOutput(queryInput);
String requestedPath = null;
List<FILE> files = listFiles(table, queryInput.getBackend(), requestedPath);
String requestedPath = null;
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if this is a query for a single file name, then get that file name in the requestedPath param for the listFiles call //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(queryInput.getFilter() != null)
{
for(QFilterCriteria criteria : CollectionUtils.nonNullList(queryInput.getFilter().getCriteria()))
{
if(criteria.getFieldName().equals(tableDetails.getFileNameFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
{
requestedPath = ValueUtils.getValueAsString(criteria.getValues().get(0));
}
}
}
List<FILE> files = listFiles(table, queryInput.getBackend(), requestedPath);
switch(tableDetails.getCardinality())
{
@ -632,7 +658,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
try
{
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
writeFile(backend, table, record, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
output.addRecord(record);
}

View File

@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -204,7 +205,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
** Write a file - to be implemented in module-specific subclasses.
*******************************************************************************/
@Override
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
public void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
{
FileUtils.writeByteArrayToFile(new File(path), contents);
}

View File

@ -363,7 +363,7 @@ public class FilesystemImporterStep implements BackendStep
path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
LOG.info("Archiving file", logPair("path", path), logPair("archiveBackendName", archiveBackend.getName()), logPair("archiveTableName", archiveTable.getName()));
archiveActionBase.writeFile(archiveBackend, path, bytes);
archiveActionBase.writeFile(archiveBackend, archiveTable, null, path, bytes);
return (path);
}

View File

@ -111,10 +111,10 @@ public class FilesystemSyncStep implements BackendStep
byte[] bytes = inputStream.readAllBytes();
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
archiveActionBase.writeFile(archiveBackend, archiveTable, null, archivePath + File.separator + sourceFileName, bytes);
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
processingActionBase.writeFile(processingBackend, processingTable, null, processingPath + File.separator + sourceFileName, bytes);
syncedFileCount++;
if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)

View File

@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLConnection;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
@ -34,6 +36,7 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -42,6 +45,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -195,14 +199,27 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
** Write a file - to be implemented in module-specific subclasses.
*******************************************************************************/
@Override
public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
public void writeFile(QBackendMetaData backendMetaData, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
{
String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
try
{
path = stripLeadingSlash(stripDuplicatedSlashes(path));
getS3Utils().writeFile(bucketName, path, contents);
String contentType = null;
if(table.getBackendDetails() instanceof S3TableBackendDetails s3TableBackendDetails)
{
contentType = switch(Objects.requireNonNullElse(s3TableBackendDetails.getContentTypeStrategy(), S3TableBackendDetails.ContentTypeStrategy.NONE))
{
case BASED_ON_FILE_NAME -> URLConnection.guessContentTypeFromName(path);
case FROM_FIELD -> record == null ? null : record.getValueString(s3TableBackendDetails.getContentTypeFieldName());
case HARDCODED -> s3TableBackendDetails.getHardcodedContentType();
case NONE -> null;
};
}
getS3Utils().writeFile(bucketName, path, contents, contentType);
}
catch(Exception e)
{
@ -277,5 +294,4 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
getS3Utils().moveObject(bucketName, source, destination);
}
}

View File

@ -22,6 +22,11 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
@ -31,6 +36,21 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
*******************************************************************************/
public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
{
private ContentTypeStrategy contentTypeStrategy = ContentTypeStrategy.NONE;
private String contentTypeFieldName;
private String hardcodedContentType;
/***************************************************************************
**
***************************************************************************/
public enum ContentTypeStrategy
{
BASED_ON_FILE_NAME,
FROM_FIELD,
HARDCODED,
NONE
}
@ -43,4 +63,135 @@ public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
setBackendType(S3BackendModule.class);
}
/***************************************************************************
**
***************************************************************************/
@Override
public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
{
super.validate(qInstance, table, qInstanceValidator);
String prefix = "Table " + (table == null ? "null" : table.getName()) + " backend details - ";
switch (Objects.requireNonNullElse(contentTypeStrategy, ContentTypeStrategy.NONE))
{
case FROM_FIELD ->
{
qInstanceValidator.assertCondition(!StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
if(table != null && qInstanceValidator.assertCondition(StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName must be set when contentTypeStrategy is " + contentTypeStrategy))
{
qInstanceValidator.assertCondition(table.getFields().containsKey(contentTypeFieldName), prefix + "contentTypeFieldName must be a valid field name in the table");
}
}
case HARDCODED ->
{
qInstanceValidator.assertCondition(!StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy);
qInstanceValidator.assertCondition(StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType must be set when contentTypeStrategy is " + contentTypeStrategy);
}
case BASED_ON_FILE_NAME, NONE ->
{
qInstanceValidator.assertCondition(!StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy);
qInstanceValidator.assertCondition(!StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
}
default ->
{
throw new IllegalStateException("Unexpected value: " + contentTypeStrategy);
}
}
}
/*******************************************************************************
** Getter for contentTypeStrategy
*******************************************************************************/
public ContentTypeStrategy getContentTypeStrategy()
{
return (this.contentTypeStrategy);
}
/*******************************************************************************
** Setter for contentTypeStrategy
*******************************************************************************/
public void setContentTypeStrategy(ContentTypeStrategy contentTypeStrategy)
{
this.contentTypeStrategy = contentTypeStrategy;
}
/*******************************************************************************
** Fluent setter for contentTypeStrategy
*******************************************************************************/
public S3TableBackendDetails withContentTypeStrategy(ContentTypeStrategy contentTypeStrategy)
{
this.contentTypeStrategy = contentTypeStrategy;
return (this);
}
/*******************************************************************************
** Getter for contentTypeFieldName
*******************************************************************************/
public String getContentTypeFieldName()
{
return (this.contentTypeFieldName);
}
/*******************************************************************************
** Setter for contentTypeFieldName
*******************************************************************************/
public void setContentTypeFieldName(String contentTypeFieldName)
{
this.contentTypeFieldName = contentTypeFieldName;
}
/*******************************************************************************
** Fluent setter for contentTypeFieldName
*******************************************************************************/
public S3TableBackendDetails withContentTypeFieldName(String contentTypeFieldName)
{
this.contentTypeFieldName = contentTypeFieldName;
return (this);
}
/*******************************************************************************
** Getter for hardcodedContentType
*******************************************************************************/
public String getHardcodedContentType()
{
return (this.hardcodedContentType);
}
/*******************************************************************************
** Setter for hardcodedContentType
*******************************************************************************/
public void setHardcodedContentType(String hardcodedContentType)
{
this.hardcodedContentType = hardcodedContentType;
}
/*******************************************************************************
** Fluent setter for hardcodedContentType
*******************************************************************************/
public S3TableBackendDetails withHardcodedContentType(String hardcodedContentType)
{
this.hardcodedContentType = hardcodedContentType;
return (this);
}
}

View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URI;
import java.net.URLEncoder;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
@ -175,7 +176,7 @@ public class S3Utils
///////////////////////////////////////////
// skip files that do not match the glob //
///////////////////////////////////////////
if(!pathMatcher.matches(Path.of(URI.create("file:///" + key))))
if(!pathMatcher.matches(Path.of(URI.create("file:///" + URLEncoder.encode(key)))))
{
// LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
continue;
@ -204,10 +205,11 @@ public class S3Utils
/*******************************************************************************
** Write a file
*******************************************************************************/
public void writeFile(String bucket, String key, byte[] contents)
public void writeFile(String bucket, String key, byte[] contents, String contentType)
{
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(contents.length);
objectMetadata.setContentType(contentType);
getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), objectMetadata);
}

View File

@ -325,20 +325,51 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
fullPath = "." + fullPath;
}
for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath))
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// in case we were asked to list a single file name, make sure we don't put a slash on the end (which wouldn't be found) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(fullPath.endsWith("/"))
{
if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename()))
{
continue;
}
fullPath = fullPath.substring(0, fullPath.length() - 1);
}
if(dirEntry.getAttributes().isDirectory())
{
// todo - recursive??
continue;
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// make a 'stat' call, to find out if the path is found, and if it is, if it describes a single file, or a directory //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
SftpClient.Attributes stat = sftpClient.stat(fullPath);
if(stat == null)
{
return (rs);
}
else if(stat.isRegularFile())
{
///////////////////////////////////////////////////////////////////////////
// split up the fullPath into its directory prefix, and file name suffix //
///////////////////////////////////////////////////////////////////////////
int lastSlashIndex = fullPath.lastIndexOf("/");
String directory = lastSlashIndex == -1 ? "./" : fullPath.substring(0, lastSlashIndex);
String fileBaseName = lastSlashIndex == -1 ? fullPath : fullPath.substring(lastSlashIndex + 1);
rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
SftpClient.DirEntry dirEntry = new SftpClient.DirEntry(fileBaseName, fullPath, stat);
rs.add(new SFTPDirEntryWithPath(directory, dirEntry));
}
else if(stat.isDirectory())
{
for(SftpClient.DirEntry dirEntry : sftpClient.readEntries(fullPath))
{
if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename()))
{
continue;
}
if(dirEntry.getAttributes().isDirectory())
{
// todo - recursive??
continue;
}
rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
}
}
return (rs);
@ -372,7 +403,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
**
***************************************************************************/
@Override
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
public void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
{
sftpClient.put(new ByteArrayInputStream(contents), path);
}

View File

@ -362,6 +362,7 @@ public class TestUtils
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new S3TableBackendDetails()
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.BASED_ON_FILE_NAME)
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")

View File

@ -229,7 +229,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
String fullPath = actionBase.getFullBasePath(table, backend);
actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes());
actionBase.writeFile(backend, table, null, fullPath + "/" + name, content.getBytes());
}

View File

@ -66,7 +66,7 @@ public class BaseS3Test extends BaseTest
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB 3.md", "# Hi, MD"); // this one, with a space in the name, tripped up listObjectsInBucketMatchingGlob's path matching at one time
amazonS3.createBucket(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND);
amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-1.txt", "Hello, Blob");

View File

@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@ -34,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
@ -54,18 +57,15 @@ public class S3InsertActionTest extends BaseS3Test
@Test
public void testCardinalityOne() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
S3InsertAction insertAction = new S3InsertAction();
insertAction.setS3Utils(getS3Utils());
InsertOutput insertOutput = insertAction.execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
@ -73,6 +73,65 @@ public class S3InsertActionTest extends BaseS3Test
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
List<String> lines = IOUtils.readLines(object.getObjectContent(), StandardCharsets.UTF_8);
assertEquals("Hi, Bob.", lines.get(0));
ObjectMetadata objectMetadata = object.getObjectMetadata();
assertEquals("text/plain", objectMetadata.getContentType());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testContentTypeFromField() throws QException
{
((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
.getBackendDetails())
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.FROM_FIELD)
.withContentTypeFieldName("contentType");
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contentType", "myContentType/fake").withValue("contents", "Hi, Bob.")));
S3InsertAction insertAction = new S3InsertAction();
insertAction.setS3Utils(getS3Utils());
InsertOutput insertOutput = insertAction.execute(insertInput);
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
ObjectMetadata objectMetadata = object.getObjectMetadata();
assertEquals("myContentType/fake", objectMetadata.getContentType());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testContentTypeHardcoded() throws QException
{
((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
.getBackendDetails())
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.HARDCODED)
.withHardcodedContentType("your-content-type");
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
S3InsertAction insertAction = new S3InsertAction();
insertAction.setS3Utils(getS3Utils());
InsertOutput insertOutput = insertAction.execute(insertInput);
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
ObjectMetadata objectMetadata = object.getObjectMetadata();
assertEquals("your-content-type", objectMetadata.getContentType());
}

View File

@ -62,6 +62,23 @@ public class S3QueryActionTest extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testGet() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3)
.withFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
Assertions.assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows from query");
}
/*******************************************************************************
**
*******************************************************************************/

View File

@ -0,0 +1,177 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import java.util.List;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import org.assertj.core.api.CollectionAssert;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for S3TableBackendDetails
*******************************************************************************/
class S3TableBackendDetailsTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testValidateContentTypeStrategyBasedOnFileNameOrNone()
{
/////////////////////////////////////////////
// same validation rules for both of these //
/////////////////////////////////////////////
for(S3TableBackendDetails.ContentTypeStrategy contentTypeStrategy : ListBuilder.of(null, S3TableBackendDetails.ContentTypeStrategy.BASED_ON_FILE_NAME, S3TableBackendDetails.ContentTypeStrategy.NONE))
{
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
.withContentTypeStrategy(contentTypeStrategy);
QTableMetaData table = getQTableMetaData();
List<String> errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.isEmpty();
s3TableBackendDetails.setHardcodedContentType("Test");
s3TableBackendDetails.setContentTypeFieldName("Test");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(2)
.contains("Table testTable backend details - contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy)
.contains("Table testTable backend details - hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testValidateContentTypeStrategyFromField()
{
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.FROM_FIELD);
QTableMetaData table = getQTableMetaData();
List<String> errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - contentTypeFieldName must be set when contentTypeStrategy is FROM_FIELD");
s3TableBackendDetails.setContentTypeFieldName("notAField");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - contentTypeFieldName must be a valid field name in the table");
table.addField(new QFieldMetaData("contentType", QFieldType.STRING));
s3TableBackendDetails.setContentTypeFieldName("contentType");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.isEmpty();
s3TableBackendDetails.setHardcodedContentType("hard");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - hardcodedContentType should not be set when contentTypeStrategy is FROM_FIELD");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testValidateContentTypeStrategyHardcoded()
{
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.HARDCODED);
QTableMetaData table = getQTableMetaData();
List<String> errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - hardcodedContentType must be set when contentTypeStrategy is HARDCODED");
s3TableBackendDetails.setHardcodedContentType("Test");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.isEmpty();
s3TableBackendDetails.setContentTypeFieldName("aField");
errors = runValidation(s3TableBackendDetails, table);
CollectionAssert.assertThatCollection(errors)
.hasSize(1)
.contains("Table testTable backend details - contentTypeFieldName should not be set when contentTypeStrategy is HARDCODED");
}
/***************************************************************************
**
***************************************************************************/
private static QTableMetaData getQTableMetaData()
{
QTableMetaData table = new QTableMetaData()
.withName("testTable")
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withField(new QFieldMetaData("fileName", QFieldType.STRING));
return table;
}
/***************************************************************************
**
***************************************************************************/
private static S3TableBackendDetails getS3TableBackendDetails()
{
S3TableBackendDetails s3TableBackendDetails = new S3TableBackendDetails()
.withContentsFieldName("contents")
.withFileNameFieldName("fileName")
.withCardinality(Cardinality.ONE);
return s3TableBackendDetails;
}
/***************************************************************************
**
***************************************************************************/
private List<String> runValidation(S3TableBackendDetails s3TableBackendDetails, QTableMetaData table)
{
QInstanceValidator validator = new QInstanceValidator();
s3TableBackendDetails.validate(QContext.getQInstance(), table, validator);
return (validator.getErrors());
}
}