Basic support for variants; more fields on ONE type file records (size, dates); apply skip, limit, filter, sort on listings/queries for ONE-type files; treat contents as heavy-field if so set; more try-catch (e.g., upon write file)

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -25,17 +25,25 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
 import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -47,12 +55,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.NotImplementedException;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -68,6 +81,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 {
    private static final QLogger LOG = QLogger.getLogger(AbstractBaseFilesystemAction.class);
 
+   protected QRecord backendVariantRecord = null;
+
 
 
    /*******************************************************************************
@@ -80,6 +95,21 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 
 
 
+   /***************************************************************************
+    ** get the size of the specified file, null if not supported/available
+    ***************************************************************************/
+   public abstract Long getFileSize(FILE file);
+
+   /***************************************************************************
+    ** get the createDate of the specified file, null if not supported/available
+    ***************************************************************************/
+   public abstract Instant getFileCreateDate(FILE file);
+
+   /***************************************************************************
+    ** get the createDate of the specified file, null if not supported/available
+    ***************************************************************************/
+   public abstract Instant getFileModifyDate(FILE file);
+
    /*******************************************************************************
     ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
     *******************************************************************************/
@@ -116,13 +146,20 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     *******************************************************************************/
    public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException;
 
+
+
    /*******************************************************************************
     ** e.g., with a base path of /foo/
     ** and a table path of /bar/
     ** and a file at /foo/bar/baz.txt
     ** give us just the baz.txt part.
     *******************************************************************************/
-   public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable);
+   public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table)
+   {
+      String tablePath    = getFullBasePath(table, backend);
+      String strippedPath = filePath.replaceFirst(".*" + tablePath, "");
+      return (strippedPath);
+   }
 
 
 
@@ -133,7 +170,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase)
    {
       AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase);
-      String                            fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : "";
+
+      String basePath = metaData.getBasePath();
+      if(backendBase.getUsesVariants())
+      {
+         Map<BackendVariantSetting, String> fieldNameMap = backendBase.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap();
+         if(fieldNameMap.containsKey(SFTPBackendVariantSetting.BASE_PATH))
+         {
+            basePath = backendVariantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.BASE_PATH));
+         }
+      }
+      String fullPath = StringUtils.hasContent(basePath) ? basePath : "";
 
       AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
       if(StringUtils.hasContent(tableDetails.getBasePath()))
@@ -208,100 +255,14 @@ public abstract class AbstractBaseFilesystemAction<FILE>
          AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
          List<FILE>                            files        = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
 
-         int recordCount = 0;
-
-         FILE_LOOP:
-         for(FILE file : files)
+         switch(tableDetails.getCardinality())
          {
-            InputStream inputStream = readFile(file);
-            switch(tableDetails.getCardinality())
-            {
-               case MANY:
-               {
-                  LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file)));
-                  switch(tableDetails.getRecordFormat())
-                  {
-                     case CSV:
-                     {
-                        String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
-                        fileContents = customizeFileContentsAfterReading(table, fileContents);
-
-                        if(queryInput.getRecordPipe() != null)
-                        {
-                           new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
-                           {
-                              ////////////////////////////////////////////////////////////////////////////////////////////
-                              // Before the records go into the pipe, make sure their backend details are added to them //
-                              ////////////////////////////////////////////////////////////////////////////////////////////
-                              addBackendDetailsToRecord(record, file);
-                           }));
-                        }
-                        else
-                        {
-                           List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
-                           addBackendDetailsToRecords(recordsInFile, file);
-                           queryOutput.addRecords(recordsInFile);
-                        }
-                        break;
-                     }
-                     case JSON:
-                     {
-                        String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
-                        fileContents = customizeFileContentsAfterReading(table, fileContents);
-
-                        // todo - pipe support!!
-                        List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
-                        addBackendDetailsToRecords(recordsInFile, file);
-
-                        queryOutput.addRecords(recordsInFile);
-                        break;
-                     }
-                     default:
-                     {
-                        throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
-                     }
-                  }
-                  break;
-               }
-               case ONE:
-               {
-                  ////////////////////////////////////////////////////////////////////////////////
-                  // for one-record tables, put the entire file's contents into a single record //
-                  ////////////////////////////////////////////////////////////////////////////////
-                  String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
-                  byte[] bytes               = inputStream.readAllBytes();
-
-                  QRecord record = new QRecord()
-                     .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
-                     .withValue(tableDetails.getContentsFieldName(), bytes);
-                  queryOutput.addRecord(record);
-
-                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  // keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) //
-                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  recordCount++;
-
-                  ////////////////////////////////////////////////////////////////////////////
-                  // break out of the file loop if we have hit the limit (if one was given) //
-                  ////////////////////////////////////////////////////////////////////////////
-                  if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null)
-                  {
-                     if(recordCount >= queryInput.getFilter().getLimit())
-                     {
-                        break FILE_LOOP;
-                     }
-                  }
-
-                  break;
-               }
-               default:
-               {
-                  throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
-               }
-            }
+            case MANY -> completeExecuteQueryForManyTable(queryInput, queryOutput, files, table, tableDetails);
+            case ONE -> completeExecuteQueryForOneTable(queryInput, queryOutput, files, table, tableDetails);
+            default -> throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
          }
 
-         return queryOutput;
+         return (queryOutput);
       }
       catch(Exception e)
       {
@@ -312,6 +273,157 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void completeExecuteQueryForOneTable(QueryInput queryInput, QueryOutput queryOutput, List<FILE> files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException
+   {
+      int           recordCount = 0;
+      List<QRecord> records     = new ArrayList<>();
+
+      for(FILE file : files)
+      {
+         ////////////////////////////////////////////////////////////////////////////////
+         // for one-record tables, put the entire file's contents into a single record //
+         ////////////////////////////////////////////////////////////////////////////////
+         String  filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
+         QRecord record              = new QRecord().withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase);
+
+         if(StringUtils.hasContent(tableDetails.getSizeFieldName()))
+         {
+            record.setValue(tableDetails.getSizeFieldName(), getFileSize(file));
+         }
+
+         if(StringUtils.hasContent(tableDetails.getCreateDateFieldName()))
+         {
+            record.setValue(tableDetails.getCreateDateFieldName(), getFileCreateDate(file));
+         }
+
+         if(StringUtils.hasContent(tableDetails.getModifyDateFieldName()))
+         {
+            record.setValue(tableDetails.getModifyDateFieldName(), getFileModifyDate(file));
+         }
+
+         if(shouldFileContentsBeRead(queryInput, table, tableDetails))
+         {
+            try(InputStream inputStream = readFile(file))
+            {
+               byte[] bytes = inputStream.readAllBytes();
+               record.withValue(tableDetails.getContentsFieldName(), bytes);
+            }
+            catch(Exception e)
+            {
+               record.addError(new SystemErrorStatusMessage("Error reading file contents: " + e.getMessage()));
+            }
+         }
+         else
+         {
+            if(StringUtils.hasContent(tableDetails.getSizeFieldName()))
+            {
+               Long size = record.getValueLong(tableDetails.getSizeFieldName());
+               if(size != null)
+               {
+                  if(record.getBackendDetails() == null)
+                  {
+                     record.setBackendDetails(new HashMap<>());
+                  }
+
+                  if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null)
+                  {
+                     record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>());
+                  }
+
+                  ((Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size);
+               }
+            }
+         }
+
+         if(BackendQueryFilterUtils.doesRecordMatch(queryInput.getFilter(), null, record))
+         {
+            records.add(record);
+         }
+      }
+
+      BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), records);
+      records = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), records);
+      queryOutput.addRecords(records);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void completeExecuteQueryForManyTable(QueryInput queryInput, QueryOutput queryOutput, List<FILE> files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException, IOException
+   {
+      int recordCount = 0;
+
+      for(FILE file : files)
+      {
+         try(InputStream inputStream = readFile(file))
+         {
+            LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file)));
+            switch(tableDetails.getRecordFormat())
+            {
+               case CSV ->
+               {
+                  String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+                  fileContents = customizeFileContentsAfterReading(table, fileContents);
+
+                  if(queryInput.getRecordPipe() != null)
+                  {
+                     new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
+                     {
+                        ////////////////////////////////////////////////////////////////////////////////////////////
+                        // Before the records go into the pipe, make sure their backend details are added to them //
+                        ////////////////////////////////////////////////////////////////////////////////////////////
+                        addBackendDetailsToRecord(record, file);
+                     }));
+                  }
+                  else
+                  {
+                     List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
+                     addBackendDetailsToRecords(recordsInFile, file);
+                     queryOutput.addRecords(recordsInFile);
+                  }
+               }
+               case JSON ->
+               {
+                  String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+                  fileContents = customizeFileContentsAfterReading(table, fileContents);
+
+                  // todo - pipe support!!
+                  List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
+                  addBackendDetailsToRecords(recordsInFile, file);
+
+                  queryOutput.addRecords(recordsInFile);
+               }
+               default -> throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
+            }
+         }
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static boolean shouldFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails)
+   {
+      boolean doReadContents = true;
+      if(table.getField(tableDetails.getContentsFieldName()).getIsHeavy())
+      {
+         if(!queryInput.getShouldFetchHeavyFields())
+         {
+            doReadContents = false;
+         }
+      }
+      return doReadContents;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -319,7 +431,16 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    {
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(countInput.getTableName());
-      queryInput.setFilter(countInput.getFilter());
+
+      QQueryFilter filter = countInput.getFilter();
+      if(filter != null)
+      {
+         filter = filter.clone();
+         filter.setSkip(null);
+         filter.setLimit(null);
+      }
+
+      queryInput.setFilter(filter);
       QueryOutput queryOutput = executeQuery(queryInput);
 
       CountOutput countOutput = new CountOutput();
@@ -353,11 +474,12 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     ** Method that subclasses can override to add pre-action things (e.g., setting up
     ** s3 client).
     *******************************************************************************/
-   public void preAction(QBackendMetaData backendMetaData)
+   public void preAction(QBackendMetaData backendMetaData) throws QException
    {
-      /////////////////////////////////////////////////////////////////////
-      // noop in base class - subclasses can add functionality if needed //
-      /////////////////////////////////////////////////////////////////////
+      if(backendMetaData.getUsesVariants())
+      {
+         this.backendVariantRecord = getVariantRecord(backendMetaData);
+      }
    }
 
 
@@ -411,10 +533,18 @@ public abstract class AbstractBaseFilesystemAction<FILE>
          {
             for(QRecord record : insertInput.getRecords())
             {
-               String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
-               writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
-               record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
-               output.addRecord(record);
+               try
+               {
+                  String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
+                  writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
+                  record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
+                  output.addRecord(record);
+               }
+               catch(Exception e)
+               {
+                  record.addError(new SystemErrorStatusMessage("Error writing file: " + e.getMessage()));
+                  output.addRecord(record);
+               }
             }
          }
          else
@@ -429,4 +559,46 @@ public abstract class AbstractBaseFilesystemAction<FILE>
          throw new QException("Error executing insert: " + e.getMessage(), e);
       }
    }
+
+
+
+   /*******************************************************************************
+    ** Get the variant id from the session for the backend.
+    *******************************************************************************/
+   protected Serializable getVariantId(QBackendMetaData backendMetaData) throws QException
+   {
+      QSession session        = QContext.getQSession();
+      String   variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
+      if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey))
+      {
+         throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"));
+      }
+      Serializable variantId = session.getBackendVariants().get(variantTypeKey);
+      return variantId;
+   }
+
+
+
+   /*******************************************************************************
+    ** For backends that use variants, look up the variant record (in theory, based
+    ** on an id in the session's backend variants map, then fetched from the backend's
+    ** variant options table.
+    *******************************************************************************/
+   protected QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException
+   {
+      Serializable variantId = getVariantId(backendMetaData);
+      GetInput     getInput  = new GetInput();
+      getInput.setShouldMaskPasswords(false);
+      getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
+      getInput.setPrimaryKey(variantId);
+      GetOutput getOutput = new GetAction().execute(getInput);
+
+      QRecord record = getOutput.getRecord();
+      if(record == null)
+      {
+         throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'"));
+      }
+      return record;
+   }
+
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java

@@ -41,6 +41,9 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
 
    private String contentsFieldName;
    private String fileNameFieldName;
+   private String sizeFieldName;
+   private String createDateFieldName;
+   private String modifyDateFieldName;
 
 
 
@@ -281,4 +284,97 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
       }
 
    }
+
+   /*******************************************************************************
+    ** Getter for sizeFieldName
+    *******************************************************************************/
+   public String getSizeFieldName()
+   {
+      return (this.sizeFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sizeFieldName
+    *******************************************************************************/
+   public void setSizeFieldName(String sizeFieldName)
+   {
+      this.sizeFieldName = sizeFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sizeFieldName
+    *******************************************************************************/
+   public AbstractFilesystemTableBackendDetails withSizeFieldName(String sizeFieldName)
+   {
+      this.sizeFieldName = sizeFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDateFieldName
+    *******************************************************************************/
+   public String getCreateDateFieldName()
+   {
+      return (this.createDateFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDateFieldName
+    *******************************************************************************/
+   public void setCreateDateFieldName(String createDateFieldName)
+   {
+      this.createDateFieldName = createDateFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDateFieldName
+    *******************************************************************************/
+   public AbstractFilesystemTableBackendDetails withCreateDateFieldName(String createDateFieldName)
+   {
+      this.createDateFieldName = createDateFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for modifyDateFieldName
+    *******************************************************************************/
+   public String getModifyDateFieldName()
+   {
+      return (this.modifyDateFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for modifyDateFieldName
+    *******************************************************************************/
+   public void setModifyDateFieldName(String modifyDateFieldName)
+   {
+      this.modifyDateFieldName = modifyDateFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for modifyDateFieldName
+    *******************************************************************************/
+   public AbstractFilesystemTableBackendDetails withModifyDateFieldName(String modifyDateFieldName)
+   {
+      this.modifyDateFieldName = modifyDateFieldName;
+      return (this);
+   }
+
+
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java

@@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
 
 
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory;
 import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule;
+import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails;
 
 
 /*******************************************************************************
@@ -64,6 +70,7 @@ public class FilesystemTableMetaDataBuilder
       {
          case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails();
          case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails();
+         case SFTPBackendModule.BACKEND_TYPE -> new SFTPTableBackendDetails();
          default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType());
       };
 
@@ -72,12 +79,27 @@ public class FilesystemTableMetaDataBuilder
          .withIsHidden(true)
          .withBackendName(backend.getName())
          .withPrimaryKeyField("fileName")
-         .withField(new QFieldMetaData("fileName", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("contents", QFieldType.STRING))
+
+         .withField(new QFieldMetaData("fileName", QFieldType.STRING))
+         .withField(new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))
+         .withField(new QFieldMetaData("contents", QFieldType.BLOB)
+            .withIsHeavy(true)
+            .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD)
+               .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "File Contents")))
+
+         .withSection(SectionFactory.defaultT1("fileName"))
+         .withSection(SectionFactory.defaultT2("contents", "size"))
+         .withSection(SectionFactory.defaultT3("createDate", "modifyDate"))
+
          .withBackendDetails(tableBackendDetails
             .withCardinality(Cardinality.ONE)
             .withFileNameFieldName("fileName")
             .withContentsFieldName("contents")
+            .withSizeFieldName("size")
+            .withCreateDateFieldName("createDate")
+            .withModifyDateFieldName("modifyDate")
             .withBasePath(basePath)
             .withGlob(glob));
    }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -35,6 +35,8 @@ import java.nio.file.Path;
 import java.nio.file.PathMatcher;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -61,6 +63,50 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
 
 
 
+   /***************************************************************************
+    *
+    ***************************************************************************/
+   @Override
+   public Long getFileSize(File file)
+   {
+      return (file.length());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Instant getFileCreateDate(File file)
+   {
+      try
+      {
+         Path                path         = file.toPath();
+         BasicFileAttributes attrs        = Files.readAttributes(path, BasicFileAttributes.class);
+         FileTime            creationTime = attrs.creationTime();
+         return creationTime.toInstant();
+      }
+      catch(IOException e)
+      {
+         LOG.warn("Error getting file createDate", e, logPair("file", file));
+         return (null);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Instant getFileModifyDate(File file)
+   {
+      return Instant.ofEpochMilli(file.lastModified());
+   }
+
+
+
    /*******************************************************************************
     ** List the files for this table.
     *******************************************************************************/

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java

@@ -63,7 +63,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
-import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -314,7 +313,7 @@ public class FilesystemImporterStep implements BackendStep
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException
+   private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws QException
    {
       if(removeFileAfterImport)
       {

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
 
 
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
@@ -35,6 +36,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3CountAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
@@ -112,6 +114,17 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public CountInterface getCountInterface()
+   {
+      return new S3CountAction();
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Instant;
 import java.util.List;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
@@ -56,11 +57,44 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Long getFileSize(S3ObjectSummary s3ObjectSummary)
+   {
+      return (s3ObjectSummary.getSize());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Instant getFileCreateDate(S3ObjectSummary s3ObjectSummary)
+   {
+      return null;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Instant getFileModifyDate(S3ObjectSummary s3ObjectSummary)
+   {
+      return s3ObjectSummary.getLastModified().toInstant();
+   }
+
+
+
    /*******************************************************************************
     ** Setup the s3 utils object to be used for this action.
     *******************************************************************************/
    @Override
-   public void preAction(QBackendMetaData backendMetaData)
+   public void preAction(QBackendMetaData backendMetaData) throws QException
    {
       super.preAction(backendMetaData);