mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-20 22:18:43 +00:00
Compare commits
18 Commits
snapshot-f
...
dependabot
Author | SHA1 | Date | |
---|---|---|---|
1599313b75 | |||
a4ffe815b5 | |||
3f75add3ed | |||
6f1e9413f6 | |||
64278e674b | |||
2fa829658f | |||
8f751d81fe | |||
d42b67582a | |||
942134b4b0 | |||
aca8436c56 | |||
94631585ee | |||
96c539b323 | |||
235cf9e16c | |||
d733ce9566 | |||
ebd9dc9c2c | |||
12e194fc2e | |||
55d046cd86 | |||
16cedfeb6e |
@ -65,7 +65,11 @@
|
||||
<artifactId>aws-java-sdk-secretsmanager</artifactId>
|
||||
<version>1.12.385</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.ibm.icu</groupId>
|
||||
<artifactId>icu4j</artifactId>
|
||||
<version>77.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
@ -115,7 +119,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<version>5.2.5</version>
|
||||
<version>5.4.0</version>
|
||||
</dependency>
|
||||
|
||||
<!-- adding to help FastExcel -->
|
||||
@ -151,16 +155,21 @@
|
||||
<version>2.3</version>
|
||||
</dependency>
|
||||
|
||||
<!-- the next 2 deps are for html to pdf - per https://www.baeldung.com/java-html-to-pdf -->
|
||||
<!-- the next 3 deps are for html to pdf -->
|
||||
<dependency>
|
||||
<groupId>org.jsoup</groupId>
|
||||
<artifactId>jsoup</artifactId>
|
||||
<version>1.15.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.xhtmlrenderer</groupId>
|
||||
<artifactId>flying-saucer-pdf-openpdf</artifactId>
|
||||
<version>9.1.22</version>
|
||||
<groupId>com.openhtmltopdf</groupId>
|
||||
<artifactId>openhtmltopdf-core</artifactId>
|
||||
<version>1.0.10</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.openhtmltopdf</groupId>
|
||||
<artifactId>openhtmltopdf-pdfbox</artifactId>
|
||||
<version>1.0.10</version>
|
||||
</dependency>
|
||||
|
||||
<!-- the next 3 deps are being added for google drive support -->
|
||||
|
@ -26,22 +26,31 @@ import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.templates.ConvertHtmlToPdfInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.templates.ConvertHtmlToPdfOutput;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.openhtmltopdf.css.constants.IdentValue;
|
||||
import com.openhtmltopdf.pdfboxout.PdfBoxFontResolver;
|
||||
import com.openhtmltopdf.pdfboxout.PdfBoxRenderer;
|
||||
import com.openhtmltopdf.pdfboxout.PdfRendererBuilder;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.xhtmlrenderer.layout.SharedContext;
|
||||
import org.xhtmlrenderer.pdf.ITextRenderer;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Action to convert a string of HTML to a PDF!
|
||||
**
|
||||
** Much credit to https://www.baeldung.com/java-html-to-pdf
|
||||
*******************************************************************************/
|
||||
**
|
||||
** Updated in March 2025 to go from flying-saucer-pdf-openpdf lib to openhtmltopdf,
|
||||
** mostly to get support for max-height on images...
|
||||
********************************************************************************/
|
||||
public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlToPdfInput, ConvertHtmlToPdfOutput>
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(ConvertHtmlToPdfAction.class);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
@ -62,31 +71,32 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
|
||||
//////////////////////////////
|
||||
// convert the XHTML to PDF //
|
||||
//////////////////////////////
|
||||
ITextRenderer renderer = new ITextRenderer();
|
||||
SharedContext sharedContext = renderer.getSharedContext();
|
||||
sharedContext.setPrint(true);
|
||||
sharedContext.setInteractive(false);
|
||||
PdfRendererBuilder builder = new PdfRendererBuilder();
|
||||
builder.toStream(input.getOutputStream());
|
||||
builder.useFastMode();
|
||||
builder.withHtmlContent(document.html(), input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
|
||||
|
||||
if(input.getBasePath() != null)
|
||||
try(PdfBoxRenderer pdfBoxRenderer = builder.buildPdfRenderer())
|
||||
{
|
||||
String baseUrl = input.getBasePath().toUri().toURL().toString();
|
||||
renderer.setDocumentFromString(document.html(), baseUrl);
|
||||
}
|
||||
else
|
||||
{
|
||||
renderer.setDocumentFromString(document.html());
|
||||
}
|
||||
pdfBoxRenderer.layout();
|
||||
pdfBoxRenderer.getSharedContext().setPrint(true);
|
||||
pdfBoxRenderer.getSharedContext().setInteractive(false);
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// register any custom fonts the input supplied //
|
||||
//////////////////////////////////////////////////
|
||||
for(Map.Entry<String, Path> entry : CollectionUtils.nonNullMap(input.getCustomFonts()).entrySet())
|
||||
{
|
||||
renderer.getFontResolver().addFont(entry.getValue().toAbsolutePath().toString(), entry.getKey(), "UTF-8", true, null);
|
||||
}
|
||||
for(Map.Entry<String, Path> entry : CollectionUtils.nonNullMap(input.getCustomFonts()).entrySet())
|
||||
{
|
||||
LOG.warn("Note: Custom fonts appear to not be working in this class at this time...");
|
||||
pdfBoxRenderer.getFontResolver().addFont(
|
||||
entry.getValue().toAbsolutePath().toFile(), // Path to the TrueType font file
|
||||
entry.getKey(), // Font family name to use in CSS
|
||||
400, // Font weight (e.g., 400 for normal, 700 for bold)
|
||||
IdentValue.NORMAL, // Font style (e.g., NORMAL, ITALIC)
|
||||
true, // Whether to subset the font
|
||||
PdfBoxFontResolver.FontGroup.MAIN // ??
|
||||
);
|
||||
}
|
||||
|
||||
renderer.layout();
|
||||
renderer.createPDF(input.getOutputStream());
|
||||
pdfBoxRenderer.createPDF();
|
||||
}
|
||||
|
||||
return (output);
|
||||
}
|
||||
|
@ -468,7 +468,7 @@ public class QRecord implements Serializable
|
||||
*******************************************************************************/
|
||||
public String getValueString(String fieldName)
|
||||
{
|
||||
return (ValueUtils.getValueAsString(values.get(fieldName)));
|
||||
return (ValueUtils.getValueAsString(getValue(fieldName)));
|
||||
}
|
||||
|
||||
|
||||
@ -479,7 +479,7 @@ public class QRecord implements Serializable
|
||||
*******************************************************************************/
|
||||
public Integer getValueInteger(String fieldName)
|
||||
{
|
||||
return (ValueUtils.getValueAsInteger(values.get(fieldName)));
|
||||
return (ValueUtils.getValueAsInteger(getValue(fieldName)));
|
||||
}
|
||||
|
||||
|
||||
@ -490,7 +490,7 @@ public class QRecord implements Serializable
|
||||
*******************************************************************************/
|
||||
public Long getValueLong(String fieldName)
|
||||
{
|
||||
return (ValueUtils.getValueAsLong(values.get(fieldName)));
|
||||
return (ValueUtils.getValueAsLong(getValue(fieldName)));
|
||||
}
|
||||
|
||||
|
||||
@ -500,7 +500,7 @@ public class QRecord implements Serializable
|
||||
*******************************************************************************/
|
||||
public BigDecimal getValueBigDecimal(String fieldName)
|
||||
{
|
||||
return (ValueUtils.getValueAsBigDecimal(values.get(fieldName)));
|
||||
return (ValueUtils.getValueAsBigDecimal(getValue(fieldName)));
|
||||
}
|
||||
|
||||
|
||||
@ -510,7 +510,7 @@ public class QRecord implements Serializable
|
||||
*******************************************************************************/
|
||||
public Boolean getValueBoolean(String fieldName)
|
||||
{
|
||||
return (ValueUtils.getValueAsBoolean(values.get(fieldName)));
|
||||
return (ValueUtils.getValueAsBoolean(getValue(fieldName)));
|
||||
}
|
||||
|
||||
|
||||
@ -520,7 +520,7 @@ public class QRecord implements Serializable
|
||||
*******************************************************************************/
|
||||
public LocalTime getValueLocalTime(String fieldName)
|
||||
{
|
||||
return (ValueUtils.getValueAsLocalTime(values.get(fieldName)));
|
||||
return (ValueUtils.getValueAsLocalTime(getValue(fieldName)));
|
||||
}
|
||||
|
||||
|
||||
@ -530,7 +530,7 @@ public class QRecord implements Serializable
|
||||
*******************************************************************************/
|
||||
public LocalDate getValueLocalDate(String fieldName)
|
||||
{
|
||||
return (ValueUtils.getValueAsLocalDate(values.get(fieldName)));
|
||||
return (ValueUtils.getValueAsLocalDate(getValue(fieldName)));
|
||||
}
|
||||
|
||||
|
||||
@ -540,7 +540,7 @@ public class QRecord implements Serializable
|
||||
*******************************************************************************/
|
||||
public byte[] getValueByteArray(String fieldName)
|
||||
{
|
||||
return (ValueUtils.getValueAsByteArray(values.get(fieldName)));
|
||||
return (ValueUtils.getValueAsByteArray(getValue(fieldName)));
|
||||
}
|
||||
|
||||
|
||||
@ -550,7 +550,7 @@ public class QRecord implements Serializable
|
||||
*******************************************************************************/
|
||||
public Instant getValueInstant(String fieldName)
|
||||
{
|
||||
return (ValueUtils.getValueAsInstant(values.get(fieldName)));
|
||||
return (ValueUtils.getValueAsInstant(getValue(fieldName)));
|
||||
}
|
||||
|
||||
|
||||
|
@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.data;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.BiFunction;
|
||||
|
||||
@ -198,4 +199,62 @@ public class QRecordWithJoinedRecords extends QRecord
|
||||
return (rs);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public Map<String, List<QRecord>> getAssociatedRecords()
|
||||
{
|
||||
return mainRecord.getAssociatedRecords();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public QRecord withAssociatedRecord(String name, QRecord associatedRecord)
|
||||
{
|
||||
mainRecord.withAssociatedRecord(name, associatedRecord);
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public QRecord withAssociatedRecords(Map<String, List<QRecord>> associatedRecords)
|
||||
{
|
||||
mainRecord.withAssociatedRecords(associatedRecords);
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public void setAssociatedRecords(Map<String, List<QRecord>> associatedRecords)
|
||||
{
|
||||
mainRecord.setAssociatedRecords(associatedRecords);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public QRecord withAssociatedRecords(String name, List<QRecord> associatedRecords)
|
||||
{
|
||||
mainRecord.withAssociatedRecords(name, associatedRecords);
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -82,6 +82,7 @@ public class HelpContentMetaDataProvider
|
||||
table.getField("key").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
|
||||
table.getField("content").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
|
||||
table.getField("content").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("html")));
|
||||
table.getField("content").withGridColumns(12);
|
||||
|
||||
if(backendDetailEnricher != null)
|
||||
{
|
||||
|
@ -26,6 +26,7 @@ import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import com.ibm.icu.text.Transliterator;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -462,6 +463,17 @@ public class StringUtils
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public static String replaceNonAsciiCharacters(String s)
|
||||
{
|
||||
Transliterator transliterator = Transliterator.getInstance("Any-Latin; Latin-ASCII");
|
||||
return (transliterator.transliterate(s));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
|
@ -22,6 +22,7 @@
|
||||
package com.kingsrook.qqq.backend.core.utils;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import org.apache.logging.log4j.Level;
|
||||
|
||||
@ -79,9 +80,36 @@ public class Timer
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void mark(String message)
|
||||
{
|
||||
mark(message, false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void mark(String message, boolean prettyPrint)
|
||||
{
|
||||
long now = System.currentTimeMillis();
|
||||
LOG.log(level, String.format("%s: Last [%5d] Total [%5d] %s", name, (now - last), (now - start), message));
|
||||
|
||||
if(!prettyPrint)
|
||||
{
|
||||
LOG.log(level, String.format("%s: Last [%5d] Total [%5d] %s", name, (now - last), (now - start), message));
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
Duration lastDuration = Duration.ofMillis(now - last);
|
||||
Duration totalDuration = Duration.ofMillis(now - start);
|
||||
|
||||
LOG.log(level, String.format(
|
||||
"%s: Last [%d hours, %d minutes, %d seconds, %d milliseconds] Total [%d hours, %d minutes, %d seconds, %d milliseconds] %s",
|
||||
name, lastDuration.toHours(), lastDuration.toMinutesPart(), lastDuration.toSecondsPart(), lastDuration.toMillisPart(),
|
||||
totalDuration.toHours(), totalDuration.toMinutesPart(), totalDuration.toSecondsPart(), totalDuration.toMillisPart(),
|
||||
message));
|
||||
}
|
||||
|
||||
last = now;
|
||||
}
|
||||
}
|
||||
|
@ -59,9 +59,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSett
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
|
||||
@ -124,10 +126,19 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
||||
*******************************************************************************/
|
||||
public abstract InputStream readFile(FILE file) throws IOException;
|
||||
|
||||
/***************************************************************************
|
||||
** Legacy signature for this method - before table & record params were added.
|
||||
***************************************************************************/
|
||||
@Deprecated(since = "call the overload that takes table and record")
|
||||
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
|
||||
{
|
||||
writeFile(backend, null, null, path, contents);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Write a file - to be implemented in module-specific subclasses.
|
||||
*******************************************************************************/
|
||||
public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException;
|
||||
public abstract void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException;
|
||||
|
||||
/*******************************************************************************
|
||||
** Get a string that represents the full path to a file.
|
||||
@ -287,8 +298,23 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
||||
|
||||
QueryOutput queryOutput = new QueryOutput(queryInput);
|
||||
|
||||
String requestedPath = null;
|
||||
List<FILE> files = listFiles(table, queryInput.getBackend(), requestedPath);
|
||||
String requestedPath = null;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if this is a query for a single file name, then get that file name in the requestedPath param for the listFiles call //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(queryInput.getFilter() != null)
|
||||
{
|
||||
for(QFilterCriteria criteria : CollectionUtils.nonNullList(queryInput.getFilter().getCriteria()))
|
||||
{
|
||||
if(criteria.getFieldName().equals(tableDetails.getFileNameFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
|
||||
{
|
||||
requestedPath = ValueUtils.getValueAsString(criteria.getValues().get(0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<FILE> files = listFiles(table, queryInput.getBackend(), requestedPath);
|
||||
|
||||
switch(tableDetails.getCardinality())
|
||||
{
|
||||
@ -632,7 +658,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
||||
try
|
||||
{
|
||||
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
|
||||
writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
|
||||
writeFile(backend, table, record, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
|
||||
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
|
||||
output.addRecord(record);
|
||||
}
|
||||
|
@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
@ -204,7 +205,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
|
||||
** Write a file - to be implemented in module-specific subclasses.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
|
||||
public void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
|
||||
{
|
||||
FileUtils.writeByteArrayToFile(new File(path), contents);
|
||||
}
|
||||
|
@ -363,7 +363,7 @@ public class FilesystemImporterStep implements BackendStep
|
||||
path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
|
||||
|
||||
LOG.info("Archiving file", logPair("path", path), logPair("archiveBackendName", archiveBackend.getName()), logPair("archiveTableName", archiveTable.getName()));
|
||||
archiveActionBase.writeFile(archiveBackend, path, bytes);
|
||||
archiveActionBase.writeFile(archiveBackend, archiveTable, null, path, bytes);
|
||||
|
||||
return (path);
|
||||
}
|
||||
|
@ -111,10 +111,10 @@ public class FilesystemSyncStep implements BackendStep
|
||||
byte[] bytes = inputStream.readAllBytes();
|
||||
|
||||
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
|
||||
archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
|
||||
archiveActionBase.writeFile(archiveBackend, archiveTable, null, archivePath + File.separator + sourceFileName, bytes);
|
||||
|
||||
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
|
||||
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
|
||||
processingActionBase.writeFile(processingBackend, processingTable, null, processingPath + File.separator + sourceFileName, bytes);
|
||||
syncedFileCount++;
|
||||
|
||||
if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
|
||||
|
@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URLConnection;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
@ -34,6 +36,7 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
@ -42,6 +45,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
@ -195,14 +199,27 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
|
||||
** Write a file - to be implemented in module-specific subclasses.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
|
||||
public void writeFile(QBackendMetaData backendMetaData, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
|
||||
{
|
||||
String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
|
||||
|
||||
try
|
||||
{
|
||||
path = stripLeadingSlash(stripDuplicatedSlashes(path));
|
||||
getS3Utils().writeFile(bucketName, path, contents);
|
||||
String contentType = null;
|
||||
|
||||
if(table.getBackendDetails() instanceof S3TableBackendDetails s3TableBackendDetails)
|
||||
{
|
||||
contentType = switch(Objects.requireNonNullElse(s3TableBackendDetails.getContentTypeStrategy(), S3TableBackendDetails.ContentTypeStrategy.NONE))
|
||||
{
|
||||
case BASED_ON_FILE_NAME -> URLConnection.guessContentTypeFromName(path);
|
||||
case FROM_FIELD -> record == null ? null : record.getValueString(s3TableBackendDetails.getContentTypeFieldName());
|
||||
case HARDCODED -> s3TableBackendDetails.getHardcodedContentType();
|
||||
case NONE -> null;
|
||||
};
|
||||
}
|
||||
|
||||
getS3Utils().writeFile(bucketName, path, contents, contentType);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
@ -277,5 +294,4 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
|
||||
getS3Utils().moveObject(bucketName, source, destination);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
@ -22,6 +22,11 @@
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
|
||||
|
||||
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
|
||||
|
||||
@ -31,6 +36,21 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
|
||||
*******************************************************************************/
|
||||
public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
|
||||
{
|
||||
private ContentTypeStrategy contentTypeStrategy = ContentTypeStrategy.NONE;
|
||||
private String contentTypeFieldName;
|
||||
private String hardcodedContentType;
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public enum ContentTypeStrategy
|
||||
{
|
||||
BASED_ON_FILE_NAME,
|
||||
FROM_FIELD,
|
||||
HARDCODED,
|
||||
NONE
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -43,4 +63,135 @@ public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
|
||||
setBackendType(S3BackendModule.class);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
|
||||
{
|
||||
super.validate(qInstance, table, qInstanceValidator);
|
||||
|
||||
String prefix = "Table " + (table == null ? "null" : table.getName()) + " backend details - ";
|
||||
switch (Objects.requireNonNullElse(contentTypeStrategy, ContentTypeStrategy.NONE))
|
||||
{
|
||||
case FROM_FIELD ->
|
||||
{
|
||||
qInstanceValidator.assertCondition(!StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||
|
||||
if(table != null && qInstanceValidator.assertCondition(StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName must be set when contentTypeStrategy is " + contentTypeStrategy))
|
||||
{
|
||||
qInstanceValidator.assertCondition(table.getFields().containsKey(contentTypeFieldName), prefix + "contentTypeFieldName must be a valid field name in the table");
|
||||
}
|
||||
}
|
||||
case HARDCODED ->
|
||||
{
|
||||
qInstanceValidator.assertCondition(!StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||
qInstanceValidator.assertCondition(StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType must be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||
}
|
||||
case BASED_ON_FILE_NAME, NONE ->
|
||||
{
|
||||
qInstanceValidator.assertCondition(!StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||
qInstanceValidator.assertCondition(!StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||
}
|
||||
default ->
|
||||
{
|
||||
throw new IllegalStateException("Unexpected value: " + contentTypeStrategy);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for contentTypeStrategy
|
||||
*******************************************************************************/
|
||||
public ContentTypeStrategy getContentTypeStrategy()
|
||||
{
|
||||
return (this.contentTypeStrategy);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for contentTypeStrategy
|
||||
*******************************************************************************/
|
||||
public void setContentTypeStrategy(ContentTypeStrategy contentTypeStrategy)
|
||||
{
|
||||
this.contentTypeStrategy = contentTypeStrategy;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for contentTypeStrategy
|
||||
*******************************************************************************/
|
||||
public S3TableBackendDetails withContentTypeStrategy(ContentTypeStrategy contentTypeStrategy)
|
||||
{
|
||||
this.contentTypeStrategy = contentTypeStrategy;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for contentTypeFieldName
|
||||
*******************************************************************************/
|
||||
public String getContentTypeFieldName()
|
||||
{
|
||||
return (this.contentTypeFieldName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for contentTypeFieldName
|
||||
*******************************************************************************/
|
||||
public void setContentTypeFieldName(String contentTypeFieldName)
|
||||
{
|
||||
this.contentTypeFieldName = contentTypeFieldName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for contentTypeFieldName
|
||||
*******************************************************************************/
|
||||
public S3TableBackendDetails withContentTypeFieldName(String contentTypeFieldName)
|
||||
{
|
||||
this.contentTypeFieldName = contentTypeFieldName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for hardcodedContentType
|
||||
*******************************************************************************/
|
||||
public String getHardcodedContentType()
|
||||
{
|
||||
return (this.hardcodedContentType);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for hardcodedContentType
|
||||
*******************************************************************************/
|
||||
public void setHardcodedContentType(String hardcodedContentType)
|
||||
{
|
||||
this.hardcodedContentType = hardcodedContentType;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for hardcodedContentType
|
||||
*******************************************************************************/
|
||||
public S3TableBackendDetails withHardcodedContentType(String hardcodedContentType)
|
||||
{
|
||||
this.hardcodedContentType = hardcodedContentType;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.PathMatcher;
|
||||
@ -175,7 +176,7 @@ public class S3Utils
|
||||
///////////////////////////////////////////
|
||||
// skip files that do not match the glob //
|
||||
///////////////////////////////////////////
|
||||
if(!pathMatcher.matches(Path.of(URI.create("file:///" + key))))
|
||||
if(!pathMatcher.matches(Path.of(URI.create("file:///" + URLEncoder.encode(key)))))
|
||||
{
|
||||
// LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
|
||||
continue;
|
||||
@ -204,10 +205,11 @@ public class S3Utils
|
||||
/*******************************************************************************
|
||||
** Write a file
|
||||
*******************************************************************************/
|
||||
public void writeFile(String bucket, String key, byte[] contents)
|
||||
public void writeFile(String bucket, String key, byte[] contents, String contentType)
|
||||
{
|
||||
ObjectMetadata objectMetadata = new ObjectMetadata();
|
||||
objectMetadata.setContentLength(contents.length);
|
||||
objectMetadata.setContentType(contentType);
|
||||
getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), objectMetadata);
|
||||
}
|
||||
|
||||
|
@ -325,20 +325,51 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
|
||||
fullPath = "." + fullPath;
|
||||
}
|
||||
|
||||
for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath))
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// in case we were asked to list a single file name, make sure we don't put a slash on the end (which wouldn't be found) //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(fullPath.endsWith("/"))
|
||||
{
|
||||
if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
fullPath = fullPath.substring(0, fullPath.length() - 1);
|
||||
}
|
||||
|
||||
if(dirEntry.getAttributes().isDirectory())
|
||||
{
|
||||
// todo - recursive??
|
||||
continue;
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// make a 'stat' call, to find out if the path is found, and if it is, if it describes a single file, or a directory //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
SftpClient.Attributes stat = sftpClient.stat(fullPath);
|
||||
if(stat == null)
|
||||
{
|
||||
return (rs);
|
||||
}
|
||||
else if(stat.isRegularFile())
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// split up the fullPath into its directory prefix, and file name suffix //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
int lastSlashIndex = fullPath.lastIndexOf("/");
|
||||
String directory = lastSlashIndex == -1 ? "./" : fullPath.substring(0, lastSlashIndex);
|
||||
String fileBaseName = lastSlashIndex == -1 ? fullPath : fullPath.substring(lastSlashIndex + 1);
|
||||
|
||||
rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
|
||||
SftpClient.DirEntry dirEntry = new SftpClient.DirEntry(fileBaseName, fullPath, stat);
|
||||
rs.add(new SFTPDirEntryWithPath(directory, dirEntry));
|
||||
}
|
||||
else if(stat.isDirectory())
|
||||
{
|
||||
for(SftpClient.DirEntry dirEntry : sftpClient.readEntries(fullPath))
|
||||
{
|
||||
if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if(dirEntry.getAttributes().isDirectory())
|
||||
{
|
||||
// todo - recursive??
|
||||
continue;
|
||||
}
|
||||
|
||||
rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
|
||||
}
|
||||
}
|
||||
|
||||
return (rs);
|
||||
@ -372,7 +403,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
|
||||
**
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
|
||||
public void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
|
||||
{
|
||||
sftpClient.put(new ByteArrayInputStream(contents), path);
|
||||
}
|
||||
|
@ -362,6 +362,7 @@ public class TestUtils
|
||||
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
|
||||
.withBackendDetails(new S3TableBackendDetails()
|
||||
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.BASED_ON_FILE_NAME)
|
||||
.withBasePath("blobs")
|
||||
.withCardinality(Cardinality.ONE)
|
||||
.withFileNameFieldName("fileName")
|
||||
|
@ -229,7 +229,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
|
||||
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
|
||||
String fullPath = actionBase.getFullBasePath(table, backend);
|
||||
|
||||
actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes());
|
||||
actionBase.writeFile(backend, table, null, fullPath + "/" + name, content.getBytes());
|
||||
}
|
||||
|
||||
|
||||
|
@ -66,7 +66,7 @@ public class BaseS3Test extends BaseTest
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB 3.md", "# Hi, MD"); // this one, with a space in the name, tripped up listObjectsInBucketMatchingGlob's path matching at one time
|
||||
|
||||
amazonS3.createBucket(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND);
|
||||
amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-1.txt", "Hello, Blob");
|
||||
|
@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
|
||||
@ -34,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@ -54,18 +57,15 @@ public class S3InsertActionTest extends BaseS3Test
|
||||
@Test
|
||||
public void testCardinalityOne() throws QException, IOException
|
||||
{
|
||||
QInstance qInstance = TestUtils.defineInstance();
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
|
||||
insertInput.setRecords(List.of(
|
||||
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
|
||||
));
|
||||
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
|
||||
|
||||
S3InsertAction insertAction = new S3InsertAction();
|
||||
insertAction.setS3Utils(getS3Utils());
|
||||
|
||||
InsertOutput insertOutput = insertAction.execute(insertInput);
|
||||
|
||||
assertThat(insertOutput.getRecords())
|
||||
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
|
||||
|
||||
@ -73,6 +73,65 @@ public class S3InsertActionTest extends BaseS3Test
|
||||
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
|
||||
List<String> lines = IOUtils.readLines(object.getObjectContent(), StandardCharsets.UTF_8);
|
||||
assertEquals("Hi, Bob.", lines.get(0));
|
||||
|
||||
ObjectMetadata objectMetadata = object.getObjectMetadata();
|
||||
assertEquals("text/plain", objectMetadata.getContentType());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testContentTypeFromField() throws QException
|
||||
{
|
||||
((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
|
||||
.getBackendDetails())
|
||||
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.FROM_FIELD)
|
||||
.withContentTypeFieldName("contentType");
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
|
||||
insertInput.setRecords(List.of(
|
||||
new QRecord().withValue("fileName", "file2.txt").withValue("contentType", "myContentType/fake").withValue("contents", "Hi, Bob.")));
|
||||
|
||||
S3InsertAction insertAction = new S3InsertAction();
|
||||
insertAction.setS3Utils(getS3Utils());
|
||||
InsertOutput insertOutput = insertAction.execute(insertInput);
|
||||
|
||||
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
|
||||
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
|
||||
ObjectMetadata objectMetadata = object.getObjectMetadata();
|
||||
assertEquals("myContentType/fake", objectMetadata.getContentType());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testContentTypeHardcoded() throws QException
|
||||
{
|
||||
((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
|
||||
.getBackendDetails())
|
||||
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.HARDCODED)
|
||||
.withHardcodedContentType("your-content-type");
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
|
||||
insertInput.setRecords(List.of(
|
||||
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
|
||||
|
||||
S3InsertAction insertAction = new S3InsertAction();
|
||||
insertAction.setS3Utils(getS3Utils());
|
||||
InsertOutput insertOutput = insertAction.execute(insertInput);
|
||||
|
||||
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
|
||||
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
|
||||
ObjectMetadata objectMetadata = object.getObjectMetadata();
|
||||
assertEquals("your-content-type", objectMetadata.getContentType());
|
||||
}
|
||||
|
||||
|
||||
|
@ -62,6 +62,23 @@ public class S3QueryActionTest extends BaseS3Test
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testGet() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3)
|
||||
.withFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
|
||||
|
||||
S3QueryAction s3QueryAction = new S3QueryAction();
|
||||
s3QueryAction.setS3Utils(getS3Utils());
|
||||
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
|
||||
Assertions.assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows from query");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
@ -0,0 +1,177 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||
import org.assertj.core.api.CollectionAssert;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for S3TableBackendDetails
|
||||
*******************************************************************************/
|
||||
class S3TableBackendDetailsTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testValidateContentTypeStrategyBasedOnFileNameOrNone()
|
||||
{
|
||||
/////////////////////////////////////////////
|
||||
// same validation rules for both of these //
|
||||
/////////////////////////////////////////////
|
||||
for(S3TableBackendDetails.ContentTypeStrategy contentTypeStrategy : ListBuilder.of(null, S3TableBackendDetails.ContentTypeStrategy.BASED_ON_FILE_NAME, S3TableBackendDetails.ContentTypeStrategy.NONE))
|
||||
{
|
||||
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
|
||||
.withContentTypeStrategy(contentTypeStrategy);
|
||||
QTableMetaData table = getQTableMetaData();
|
||||
|
||||
List<String> errors = runValidation(s3TableBackendDetails, table);
|
||||
CollectionAssert.assertThatCollection(errors)
|
||||
.isEmpty();
|
||||
|
||||
s3TableBackendDetails.setHardcodedContentType("Test");
|
||||
s3TableBackendDetails.setContentTypeFieldName("Test");
|
||||
errors = runValidation(s3TableBackendDetails, table);
|
||||
CollectionAssert.assertThatCollection(errors)
|
||||
.hasSize(2)
|
||||
.contains("Table testTable backend details - contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy)
|
||||
.contains("Table testTable backend details - hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testValidateContentTypeStrategyFromField()
|
||||
{
|
||||
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
|
||||
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.FROM_FIELD);
|
||||
QTableMetaData table = getQTableMetaData();
|
||||
|
||||
List<String> errors = runValidation(s3TableBackendDetails, table);
|
||||
CollectionAssert.assertThatCollection(errors)
|
||||
.hasSize(1)
|
||||
.contains("Table testTable backend details - contentTypeFieldName must be set when contentTypeStrategy is FROM_FIELD");
|
||||
|
||||
s3TableBackendDetails.setContentTypeFieldName("notAField");
|
||||
errors = runValidation(s3TableBackendDetails, table);
|
||||
CollectionAssert.assertThatCollection(errors)
|
||||
.hasSize(1)
|
||||
.contains("Table testTable backend details - contentTypeFieldName must be a valid field name in the table");
|
||||
|
||||
table.addField(new QFieldMetaData("contentType", QFieldType.STRING));
|
||||
s3TableBackendDetails.setContentTypeFieldName("contentType");
|
||||
errors = runValidation(s3TableBackendDetails, table);
|
||||
CollectionAssert.assertThatCollection(errors)
|
||||
.isEmpty();
|
||||
|
||||
s3TableBackendDetails.setHardcodedContentType("hard");
|
||||
errors = runValidation(s3TableBackendDetails, table);
|
||||
CollectionAssert.assertThatCollection(errors)
|
||||
.hasSize(1)
|
||||
.contains("Table testTable backend details - hardcodedContentType should not be set when contentTypeStrategy is FROM_FIELD");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testValidateContentTypeStrategyHardcoded()
|
||||
{
|
||||
S3TableBackendDetails s3TableBackendDetails = getS3TableBackendDetails()
|
||||
.withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.HARDCODED);
|
||||
QTableMetaData table = getQTableMetaData();
|
||||
|
||||
List<String> errors = runValidation(s3TableBackendDetails, table);
|
||||
CollectionAssert.assertThatCollection(errors)
|
||||
.hasSize(1)
|
||||
.contains("Table testTable backend details - hardcodedContentType must be set when contentTypeStrategy is HARDCODED");
|
||||
|
||||
s3TableBackendDetails.setHardcodedContentType("Test");
|
||||
errors = runValidation(s3TableBackendDetails, table);
|
||||
CollectionAssert.assertThatCollection(errors)
|
||||
.isEmpty();
|
||||
|
||||
s3TableBackendDetails.setContentTypeFieldName("aField");
|
||||
errors = runValidation(s3TableBackendDetails, table);
|
||||
CollectionAssert.assertThatCollection(errors)
|
||||
.hasSize(1)
|
||||
.contains("Table testTable backend details - contentTypeFieldName should not be set when contentTypeStrategy is HARDCODED");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
private static QTableMetaData getQTableMetaData()
|
||||
{
|
||||
QTableMetaData table = new QTableMetaData()
|
||||
.withName("testTable")
|
||||
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
|
||||
.withField(new QFieldMetaData("fileName", QFieldType.STRING));
|
||||
return table;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
private static S3TableBackendDetails getS3TableBackendDetails()
|
||||
{
|
||||
S3TableBackendDetails s3TableBackendDetails = new S3TableBackendDetails()
|
||||
.withContentsFieldName("contents")
|
||||
.withFileNameFieldName("fileName")
|
||||
.withCardinality(Cardinality.ONE);
|
||||
return s3TableBackendDetails;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
private List<String> runValidation(S3TableBackendDetails s3TableBackendDetails, QTableMetaData table)
|
||||
{
|
||||
QInstanceValidator validator = new QInstanceValidator();
|
||||
s3TableBackendDetails.validate(QContext.getQInstance(), table, validator);
|
||||
return (validator.getErrors());
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user