From 5a7199495d6abb833d4fe62133dded69719291d8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:24:10 -0600 Subject: [PATCH] Basic support for variants; more fields on ONE type file records (size, dates); apply skip, limit, filter, sort on listings/queries for ONE-type files; treat contents as heavy-field if so set; more try-catch (e.g., upon write file) --- .../actions/AbstractBaseFilesystemAction.java | 376 +++++++++++++----- ...AbstractFilesystemTableBackendDetails.java | 96 +++++ .../FilesystemTableMetaDataBuilder.java | 26 +- .../actions/AbstractFilesystemAction.java | 46 +++ .../importer/FilesystemImporterStep.java | 3 +- .../module/filesystem/s3/S3BackendModule.java | 13 + .../s3/actions/AbstractS3Action.java | 36 +- 7 files changed, 489 insertions(+), 107 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index fdf343c6..a896230c 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -25,17 +25,25 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -47,12 +55,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -68,6 +81,8 @@ public abstract class AbstractBaseFilesystemAction { private static final QLogger LOG = QLogger.getLogger(AbstractBaseFilesystemAction.class); + protected QRecord backendVariantRecord = null; + /******************************************************************************* @@ -80,6 +95,21 @@ public abstract class AbstractBaseFilesystemAction + /*************************************************************************** + ** get the size of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Long getFileSize(FILE file); + + /*************************************************************************** + ** get the createDate of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Instant getFileCreateDate(FILE file); + + /*************************************************************************** + ** get the createDate of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Instant getFileModifyDate(FILE file); + /******************************************************************************* ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses. *******************************************************************************/ @@ -116,13 +146,20 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; + + /******************************************************************************* ** e.g., with a base path of /foo/ ** and a table path of /bar/ ** and a file at /foo/bar/baz.txt ** give us just the baz.txt part. *******************************************************************************/ - public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable); + public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); + return (strippedPath); + } @@ -133,7 +170,17 @@ public abstract class AbstractBaseFilesystemAction public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase) { AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase); - String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : ""; + + String basePath = metaData.getBasePath(); + if(backendBase.getUsesVariants()) + { + Map fieldNameMap = backendBase.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap(); + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.BASE_PATH)) + { + basePath = backendVariantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.BASE_PATH)); + } + } + String fullPath = StringUtils.hasContent(basePath) ? basePath : ""; AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); if(StringUtils.hasContent(tableDetails.getBasePath())) @@ -208,100 +255,14 @@ public abstract class AbstractBaseFilesystemAction AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); List files = listFiles(table, queryInput.getBackend(), queryInput.getFilter()); - int recordCount = 0; - - FILE_LOOP: - for(FILE file : files) + switch(tableDetails.getCardinality()) { - InputStream inputStream = readFile(file); - switch(tableDetails.getCardinality()) - { - case MANY: - { - LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file))); - switch(tableDetails.getRecordFormat()) - { - case CSV: - { - String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - fileContents = customizeFileContentsAfterReading(table, fileContents); - - if(queryInput.getRecordPipe() != null) - { - new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> - { - //////////////////////////////////////////////////////////////////////////////////////////// - // Before the records go into the pipe, make sure their backend details are added to them // - //////////////////////////////////////////////////////////////////////////////////////////// - addBackendDetailsToRecord(record, file); - })); - } - else - { - List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - queryOutput.addRecords(recordsInFile); - } - break; - } - case JSON: - { - String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - fileContents = customizeFileContentsAfterReading(table, fileContents); - - // todo - pipe support!! - List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - - queryOutput.addRecords(recordsInFile); - break; - } - default: - { - throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); - } - } - break; - } - case ONE: - { - //////////////////////////////////////////////////////////////////////////////// - // for one-record tables, put the entire file's contents into a single record // - //////////////////////////////////////////////////////////////////////////////// - String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); - byte[] bytes = inputStream.readAllBytes(); - - QRecord record = new QRecord() - .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase) - .withValue(tableDetails.getContentsFieldName(), bytes); - queryOutput.addRecord(record); - - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - // keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) // - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - recordCount++; - - //////////////////////////////////////////////////////////////////////////// - // break out of the file loop if we have hit the limit (if one was given) // - //////////////////////////////////////////////////////////////////////////// - if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null) - { - if(recordCount >= queryInput.getFilter().getLimit()) - { - break FILE_LOOP; - } - } - - break; - } - default: - { - throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); - } - } + case MANY -> completeExecuteQueryForManyTable(queryInput, queryOutput, files, table, tableDetails); + case ONE -> completeExecuteQueryForOneTable(queryInput, queryOutput, files, table, tableDetails); + default -> throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); } - return queryOutput; + return (queryOutput); } catch(Exception e) { @@ -312,6 +273,157 @@ public abstract class AbstractBaseFilesystemAction + /*************************************************************************** + ** + ***************************************************************************/ + private void completeExecuteQueryForOneTable(QueryInput queryInput, QueryOutput queryOutput, List files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException + { + int recordCount = 0; + List records = new ArrayList<>(); + + for(FILE file : files) + { + //////////////////////////////////////////////////////////////////////////////// + // for one-record tables, put the entire file's contents into a single record // + //////////////////////////////////////////////////////////////////////////////// + String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); + QRecord record = new QRecord().withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase); + + if(StringUtils.hasContent(tableDetails.getSizeFieldName())) + { + record.setValue(tableDetails.getSizeFieldName(), getFileSize(file)); + } + + if(StringUtils.hasContent(tableDetails.getCreateDateFieldName())) + { + record.setValue(tableDetails.getCreateDateFieldName(), getFileCreateDate(file)); + } + + if(StringUtils.hasContent(tableDetails.getModifyDateFieldName())) + { + record.setValue(tableDetails.getModifyDateFieldName(), getFileModifyDate(file)); + } + + if(shouldFileContentsBeRead(queryInput, table, tableDetails)) + { + try(InputStream inputStream = readFile(file)) + { + byte[] bytes = inputStream.readAllBytes(); + record.withValue(tableDetails.getContentsFieldName(), bytes); + } + catch(Exception e) + { + record.addError(new SystemErrorStatusMessage("Error reading file contents: " + e.getMessage())); + } + } + else + { + if(StringUtils.hasContent(tableDetails.getSizeFieldName())) + { + Long size = record.getValueLong(tableDetails.getSizeFieldName()); + if(size != null) + { + if(record.getBackendDetails() == null) + { + record.setBackendDetails(new HashMap<>()); + } + + if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null) + { + record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>()); + } + + ((Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size); + } + } + } + + if(BackendQueryFilterUtils.doesRecordMatch(queryInput.getFilter(), null, record)) + { + records.add(record); + } + } + + BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), records); + records = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), records); + queryOutput.addRecords(records); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void completeExecuteQueryForManyTable(QueryInput queryInput, QueryOutput queryOutput, List files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException, IOException + { + int recordCount = 0; + + for(FILE file : files) + { + try(InputStream inputStream = readFile(file)) + { + LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file))); + switch(tableDetails.getRecordFormat()) + { + case CSV -> + { + String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + if(queryInput.getRecordPipe() != null) + { + new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> + { + //////////////////////////////////////////////////////////////////////////////////////////// + // Before the records go into the pipe, make sure their backend details are added to them // + //////////////////////////////////////////////////////////////////////////////////////////// + addBackendDetailsToRecord(record, file); + })); + } + else + { + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + queryOutput.addRecords(recordsInFile); + } + } + case JSON -> + { + String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + // todo - pipe support!! + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + + queryOutput.addRecords(recordsInFile); + } + default -> throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static boolean shouldFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) + { + boolean doReadContents = true; + if(table.getField(tableDetails.getContentsFieldName()).getIsHeavy()) + { + if(!queryInput.getShouldFetchHeavyFields()) + { + doReadContents = false; + } + } + return doReadContents; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -319,7 +431,16 @@ public abstract class AbstractBaseFilesystemAction { QueryInput queryInput = new QueryInput(); queryInput.setTableName(countInput.getTableName()); - queryInput.setFilter(countInput.getFilter()); + + QQueryFilter filter = countInput.getFilter(); + if(filter != null) + { + filter = filter.clone(); + filter.setSkip(null); + filter.setLimit(null); + } + + queryInput.setFilter(filter); QueryOutput queryOutput = executeQuery(queryInput); CountOutput countOutput = new CountOutput(); @@ -353,11 +474,12 @@ public abstract class AbstractBaseFilesystemAction ** Method that subclasses can override to add pre-action things (e.g., setting up ** s3 client). *******************************************************************************/ - public void preAction(QBackendMetaData backendMetaData) + public void preAction(QBackendMetaData backendMetaData) throws QException { - ///////////////////////////////////////////////////////////////////// - // noop in base class - subclasses can add functionality if needed // - ///////////////////////////////////////////////////////////////////// + if(backendMetaData.getUsesVariants()) + { + this.backendVariantRecord = getVariantRecord(backendMetaData); + } } @@ -411,10 +533,18 @@ public abstract class AbstractBaseFilesystemAction { for(QRecord record : insertInput.getRecords()) { - String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); - writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); - record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); - output.addRecord(record); + try + { + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); + writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); + record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); + output.addRecord(record); + } + catch(Exception e) + { + record.addError(new SystemErrorStatusMessage("Error writing file: " + e.getMessage())); + output.addRecord(record); + } } } else @@ -429,4 +559,46 @@ public abstract class AbstractBaseFilesystemAction throw new QException("Error executing insert: " + e.getMessage(), e); } } + + + + /******************************************************************************* + ** Get the variant id from the session for the backend. + *******************************************************************************/ + protected Serializable getVariantId(QBackendMetaData backendMetaData) throws QException + { + QSession session = QContext.getQSession(); + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) + { + throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); + } + Serializable variantId = session.getBackendVariants().get(variantTypeKey); + return variantId; + } + + + + /******************************************************************************* + ** For backends that use variants, look up the variant record (in theory, based + ** on an id in the session's backend variants map, then fetched from the backend's + ** variant options table. + *******************************************************************************/ + protected QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException + { + Serializable variantId = getVariantId(backendMetaData); + GetInput getInput = new GetInput(); + getInput.setShouldMaskPasswords(false); + getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + getInput.setPrimaryKey(variantId); + GetOutput getOutput = new GetAction().execute(getInput); + + QRecord record = getOutput.getRecord(); + if(record == null) + { + throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); + } + return record; + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index f50f1b4c..6c9c6bc5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -41,6 +41,9 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private String contentsFieldName; private String fileNameFieldName; + private String sizeFieldName; + private String createDateFieldName; + private String modifyDateFieldName; @@ -281,4 +284,97 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails } } + + /******************************************************************************* + ** Getter for sizeFieldName + *******************************************************************************/ + public String getSizeFieldName() + { + return (this.sizeFieldName); + } + + + + /******************************************************************************* + ** Setter for sizeFieldName + *******************************************************************************/ + public void setSizeFieldName(String sizeFieldName) + { + this.sizeFieldName = sizeFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for sizeFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withSizeFieldName(String sizeFieldName) + { + this.sizeFieldName = sizeFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDateFieldName + *******************************************************************************/ + public String getCreateDateFieldName() + { + return (this.createDateFieldName); + } + + + + /******************************************************************************* + ** Setter for createDateFieldName + *******************************************************************************/ + public void setCreateDateFieldName(String createDateFieldName) + { + this.createDateFieldName = createDateFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for createDateFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withCreateDateFieldName(String createDateFieldName) + { + this.createDateFieldName = createDateFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDateFieldName + *******************************************************************************/ + public String getModifyDateFieldName() + { + return (this.modifyDateFieldName); + } + + + + /******************************************************************************* + ** Setter for modifyDateFieldName + *******************************************************************************/ + public void setModifyDateFieldName(String modifyDateFieldName) + { + this.modifyDateFieldName = modifyDateFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDateFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withModifyDateFieldName(String modifyDateFieldName) + { + this.modifyDateFieldName = modifyDateFieldName; + return (this); + } + + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java index 19c1601e..4157ffb6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java @@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory; import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; /******************************************************************************* @@ -64,6 +70,7 @@ public class FilesystemTableMetaDataBuilder { case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails(); case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails(); + case SFTPBackendModule.BACKEND_TYPE -> new SFTPTableBackendDetails(); default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType()); }; @@ -72,12 +79,27 @@ public class FilesystemTableMetaDataBuilder .withIsHidden(true) .withBackendName(backend.getName()) .withPrimaryKeyField("fileName") - .withField(new QFieldMetaData("fileName", QFieldType.INTEGER)) - .withField(new QFieldMetaData("contents", QFieldType.STRING)) + + .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB) + .withIsHeavy(true) + .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "File Contents"))) + + .withSection(SectionFactory.defaultT1("fileName")) + .withSection(SectionFactory.defaultT2("contents", "size")) + .withSection(SectionFactory.defaultT3("createDate", "modifyDate")) + .withBackendDetails(tableBackendDetails .withCardinality(Cardinality.ONE) .withFileNameFieldName("fileName") .withContentsFieldName("contents") + .withSizeFieldName("size") + .withCreateDateFieldName("createDate") + .withModifyDateFieldName("modifyDate") .withBasePath(basePath) .withGlob(glob)); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index c9fb807f..5dde1ab6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -35,6 +35,8 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -61,6 +63,50 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Long getFileSize(File file) + { + return (file.length()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileCreateDate(File file) + { + try + { + Path path = file.toPath(); + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); + FileTime creationTime = attrs.creationTime(); + return creationTime.toInstant(); + } + catch(IOException e) + { + LOG.warn("Error getting file createDate", e, logPair("file", file)); + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileModifyDate(File file) + { + return Instant.ofEpochMilli(file.lastModified()); + } + + + /******************************************************************************* ** List the files for this table. *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java index 43867e99..f41ceb6b 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java @@ -63,7 +63,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -314,7 +313,7 @@ public class FilesystemImporterStep implements BackendStep /******************************************************************************* ** *******************************************************************************/ - private static void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException + private static void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws QException { if(removeFileAfterImport) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index 8a9a6272..05610fe6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; @@ -35,6 +36,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3CountAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; @@ -112,6 +114,17 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new S3CountAction(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 91383c4b..bc0d96d5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import java.io.IOException; import java.io.InputStream; +import java.time.Instant; import java.util.List; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; @@ -56,11 +57,44 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction