From 624a723b542b364aec3e3762b5e26b7b87f3e7a9 Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Thu, 11 Jan 2024 07:46:30 -0600
Subject: [PATCH] CE-781 Initial checkin of filesystem importer meta-data
 template and process

---
 .../FilesystemTableMetaDataBuilder.java       | 199 +++++++
 .../FilesystemImporterMetaDataTemplate.java   | 502 ++++++++++++++++++
 ...esystemImporterProcessMetaDataBuilder.java | 164 ++++++
 .../importer/FilesystemImporterStep.java      | 363 +++++++++++++
 .../ImportRecordPostQueryCustomizer.java      |  82 +++
 .../backend/module/filesystem/TestUtils.java  |  58 +-
 .../importer/FilesystemImporterStepTest.java  | 114 ++++
 .../ImportRecordPostQueryCustomizerTest.java  |  85 +++
 8 files changed, 1566 insertions(+), 1 deletion(-)
 create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java
 create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java
 create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java
 create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java
 create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizer.java
 create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java
 create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizerTest.java

diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java
new file mode 100644
index 00000000..18b588e6
--- /dev/null
+++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java
@@ -0,0 +1,199 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class FilesystemTableMetaDataBuilder
+{
+   private String           name;
+   private QBackendMetaData backend;
+   private String           basePath;
+   private String           glob;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:Indentation")
+   public QTableMetaData buildStandardCardinalityOneTable()
+   {
+      AbstractFilesystemTableBackendDetails tableBackendDetails = switch(backend.getBackendType())
+      {
+         case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails();
+         case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails();
+         default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType());
+      };
+
+      return new QTableMetaData()
+         .withName(name)
+         .withIsHidden(true)
+         .withBackendName(backend.getName())
+         .withPrimaryKeyField("fileName")
+         .withField(new QFieldMetaData("fileName", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("contents", QFieldType.STRING))
+         .withBackendDetails(tableBackendDetails
+            .withCardinality(Cardinality.ONE)
+            .withFileNameFieldName("fileName")
+            .withContentsFieldName("contents")
+            .withBasePath(basePath)
+            .withGlob(glob));
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for backend
+    *******************************************************************************/
+   public QBackendMetaData getBackend()
+   {
+      return (this.backend);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backend
+    *******************************************************************************/
+   public void setBackend(QBackendMetaData backend)
+   {
+      this.backend = backend;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backend
+    *******************************************************************************/
+   public FilesystemTableMetaDataBuilder withBackend(QBackendMetaData backend)
+   {
+      this.backend = backend;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    *******************************************************************************/
+   public String getName()
+   {
+      return (this.name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    *******************************************************************************/
+   public void setName(String name)
+   {
+      this.name = name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for name
+    *******************************************************************************/
+   public FilesystemTableMetaDataBuilder withName(String name)
+   {
+      this.name = name;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for basePath
+    *******************************************************************************/
+   public String getBasePath()
+   {
+      return (this.basePath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for basePath
+    *******************************************************************************/
+   public void setBasePath(String basePath)
+   {
+      this.basePath = basePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for basePath
+    *******************************************************************************/
+   public FilesystemTableMetaDataBuilder withBasePath(String basePath)
+   {
+      this.basePath = basePath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for glob
+    *******************************************************************************/
+   public String getGlob()
+   {
+      return (this.glob);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for glob
+    *******************************************************************************/
+   public void setGlob(String glob)
+   {
+      this.glob = glob;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for glob
+    *******************************************************************************/
+   public FilesystemTableMetaDataBuilder withGlob(String glob)
+   {
+      this.glob = glob;
+      return (this);
+   }
+
+}
diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java
new file mode 100644
index 00000000..c9a68d37
--- /dev/null
+++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java
@@ -0,0 +1,502 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.util.List;
+import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+
+
+/*******************************************************************************
+ ** Class to serve as a template for producing an instance of a process & tables
+ ** that provide the QQQ service to manage importing files (e.g., partner feeds on S3).
+ **
+ ** The template contains the following components:
+ ** - A process that loads files from a source-table (e.g., of filesystem, cardinality=ONE)
+ **   and stores them in the following tables:
+ ** - {baseName}importFile table - simple header for imported files.
+ ** - {baseName}importRecord table - a record foreach record in an imported file.
+ ** - PVS for the importFile table
+ ** - Join & Widget (to show importRecords on importFile view screen)
+ **
+ ** Most likely one would add all the meta-data objects in an instance of this
+ ** template, then either use tableAutomations or a basepull process against records
+ ** in the importRecord table, to run through a process (e.g., an AbstractTableSync)
+ ** to result in final values for your business case.
+ **
+ ** A typical usage may look like:
+ **
+ ** <pre>
+ // set up the process that'll be used to import the files.
+ FilesystemImporterProcessMetaDataBuilder importerProcessBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
+ .withFileFormat("csv")
+ .withSourceTableName(MyFeedSourceTableMetaDataProducer.NAME)
+ .withRemoveFileAfterImport(true)
+ .withUpdateFileIfNameExists(false)
+ .withName("myFeedImporter")
+ .withSchedule(new QScheduleMetaData().withRepeatSeconds(300));
+
+ FilesystemImporterMetaDataTemplate template = new FilesystemImporterMetaDataTemplate(qInstance, "myFeed", MongoDBMetaDataProducer.NAME, importerProcessBuilder, table ->
+ {
+ // whatever customizations you may need on the tables
+ table.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED));
+ });
+
+ // set up automations on the table
+ template.addAutomationStatusField(template.getImportRecordTable(), getStandardAutomationStatusField().withBackendName("metaData.automationStatus"));
+ template.addStandardPostInsertAutomation(template.getImportRecordTable(), getBasicTableAutomationDetails(), "myFeedTableSyncProcess");
+
+ // finally, add all the meta-data from the template to a QInstance
+ template.addToInstance(qInstance);
+ <pre>
+ **
+ *******************************************************************************/
+public class FilesystemImporterMetaDataTemplate
+{
+   public static final String IMPORT_FILE_TABLE_SUFFIX       = "ImportFile";
+   public static final String IMPORT_RECORD_TABLE_SUFFIX     = "ImportRecord";
+   public static final String IMPORT_FILE_RECORD_JOIN_SUFFIX = "ImportFileImportRecordJoin";
+
+   private QTableMetaData                           importFileTable;
+   private QTableMetaData                           importRecordTable;
+   private QPossibleValueSource                     importFilePVS;
+   private QJoinMetaData                            importFileImportRecordJoin;
+   private QWidgetMetaDataInterface                 importFileImportRecordJoinWidget;
+   private FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate(QInstance qInstance, String importBaseName, String backendName, FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder, Consumer<QTableMetaData> tableEnricher)
+   {
+      QBackendMetaData backend = qInstance.getBackend(backendName);
+
+      this.importFileTable = defineTableImportFile(backend, importBaseName);
+      this.importRecordTable = defineTableImportRecord(backend, importBaseName);
+
+      for(QTableMetaData table : List.of(this.importFileTable, this.importRecordTable))
+      {
+         table.setBackendName(backendName);
+         if(tableEnricher != null)
+         {
+            tableEnricher.accept(table);
+         }
+      }
+
+      this.importFilePVS = QPossibleValueSource.newForTable(this.importFileTable.getName());
+
+      this.importFileImportRecordJoin = defineImportFileImportRecordJoin(importBaseName);
+      this.importFileImportRecordJoinWidget = defineImportFileImportRecordChildWidget(this.importFileImportRecordJoin);
+
+      this.importerProcessMetaDataBuilder = importerProcessMetaDataBuilder
+         .withImportFileTable(this.importFileTable.getName())
+         .withImportRecordTable(this.importRecordTable.getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addAutomationStatusField(QTableMetaData table, QFieldMetaData automationStatusField)
+   {
+      table.addField(automationStatusField);
+      table.getSections().get(1).getFieldNames().add(0, automationStatusField.getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public TableAutomationAction addStandardPostInsertAutomation(QTableMetaData table, QTableAutomationDetails automationDetails, String processName)
+   {
+      TableAutomationAction action = new TableAutomationAction()
+         .withName(table.getName() + "PostInsert")
+         .withTriggerEvent(TriggerEvent.POST_INSERT)
+         .withProcessName(processName);
+
+      table.withAutomationDetails(automationDetails
+         .withAction(action));
+
+      return (action);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QWidgetMetaDataInterface defineImportFileImportRecordChildWidget(QJoinMetaData join)
+   {
+      return ChildRecordListRenderer.widgetMetaDataBuilder(join)
+         .withName(join.getName())
+         .withLabel("Import Records")
+         .withCanAddChildRecord(false)
+         .getWidgetMetaData();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QJoinMetaData defineImportFileImportRecordJoin(String importBaseName)
+   {
+      return new QJoinMetaData()
+         .withLeftTable(importBaseName + IMPORT_FILE_TABLE_SUFFIX)
+         .withRightTable(importBaseName + IMPORT_RECORD_TABLE_SUFFIX)
+         .withName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX)
+         .withType(JoinType.ONE_TO_MANY)
+         .withJoinOn(new JoinOn("id", "importFileId"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData defineTableImportFile(QBackendMetaData backend, String importBaseName)
+   {
+      QFieldType idType = getIdFieldType(backend);
+
+      QTableMetaData qTableMetaData = new QTableMetaData()
+         .withName(importBaseName + IMPORT_FILE_TABLE_SUFFIX)
+         .withIcon(new QIcon().withName("upload_file"))
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("sourceFileName")
+         .withPrimaryKeyField("id")
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD))
+
+         .withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend)))
+         .withField(new QFieldMetaData("sourceFileName", QFieldType.STRING))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
+
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName")))
+         .withSection(new QFieldSection("records", new QIcon().withName("power_input"), Tier.T2).withWidgetName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
+
+         .withAssociation(new Association().withName("importRecords").withJoinName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX).withAssociatedTableName(importBaseName + IMPORT_RECORD_TABLE_SUFFIX));
+
+      return (qTableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QFieldType getIdFieldType(QBackendMetaData backend)
+   {
+      QFieldType idType = QFieldType.INTEGER;
+      if("mongodb".equals(backend.getBackendType()))
+      {
+         idType = QFieldType.STRING;
+      }
+      return idType;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getIdFieldBackendName(QBackendMetaData backend)
+   {
+      if("mongodb".equals(backend.getBackendType()))
+      {
+         return ("_id");
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData defineTableImportRecord(QBackendMetaData backend, String importBaseName)
+   {
+      QFieldType idType = getIdFieldType(backend);
+
+      QTableMetaData qTableMetaData = new QTableMetaData()
+         .withName(importBaseName + IMPORT_RECORD_TABLE_SUFFIX)
+         .withIcon(new QIcon().withName("power_input"))
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("importFileId", "recordNo")
+         .withPrimaryKeyField("id")
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD))
+         .withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(ImportRecordPostQueryCustomizer.class))
+
+         .withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend)))
+
+         .withField(new QFieldMetaData("importFileId", idType).withBackendName("metaData.importFileId")
+            .withPossibleValueSourceName(importBaseName + IMPORT_FILE_TABLE_SUFFIX))
+         .withField(new QFieldMetaData("recordNo", QFieldType.INTEGER).withBackendName("metaData.recordNo"))
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // so, we'll use this field as a "virtual" field, e.g., populated with JSON in table post-query customizer, with all un-structured values //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         .withField(new QFieldMetaData("values", QFieldType.TEXT)
+            .withIsEditable(false)
+            .withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR)
+               .withValue(AdornmentType.CodeEditorValues.languageMode("json"))))
+
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("metaData.createDate").withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("metaData.modifyDate").withIsEditable(false))
+
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "importFileId", "recordNo")))
+         .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("values")))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
+
+      return (qTableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addToInstance(QInstance instance)
+   {
+      instance.add(importFileTable);
+      instance.add(importRecordTable);
+      instance.add(importFilePVS);
+      instance.add(importFileImportRecordJoin);
+      instance.add(importFileImportRecordJoinWidget);
+      instance.add(importerProcessMetaDataBuilder.getProcessMetaData());
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importFileTable
+    *******************************************************************************/
+   public QTableMetaData getImportFileTable()
+   {
+      return (this.importFileTable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importFileTable
+    *******************************************************************************/
+   public void setImportFileTable(QTableMetaData importFileTable)
+   {
+      this.importFileTable = importFileTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importFileTable
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportFileTable(QTableMetaData importFileTable)
+   {
+      this.importFileTable = importFileTable;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importRecordTable
+    *******************************************************************************/
+   public QTableMetaData getImportRecordTable()
+   {
+      return (this.importRecordTable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importRecordTable
+    *******************************************************************************/
+   public void setImportRecordTable(QTableMetaData importRecordTable)
+   {
+      this.importRecordTable = importRecordTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importRecordTable
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportRecordTable(QTableMetaData importRecordTable)
+   {
+      this.importRecordTable = importRecordTable;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importFilePVS
+    *******************************************************************************/
+   public QPossibleValueSource getImportFilePVS()
+   {
+      return (this.importFilePVS);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importFilePVS
+    *******************************************************************************/
+   public void setImportFilePVS(QPossibleValueSource importFilePVS)
+   {
+      this.importFilePVS = importFilePVS;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importFilePVS
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportFilePVS(QPossibleValueSource importFilePVS)
+   {
+      this.importFilePVS = importFilePVS;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importFileImportRecordJoin
+    *******************************************************************************/
+   public QJoinMetaData getImportFileImportRecordJoin()
+   {
+      return (this.importFileImportRecordJoin);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importFileImportRecordJoin
+    *******************************************************************************/
+   public void setImportFileImportRecordJoin(QJoinMetaData importFileImportRecordJoin)
+   {
+      this.importFileImportRecordJoin = importFileImportRecordJoin;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importFileImportRecordJoin
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportFileImportRecordJoin(QJoinMetaData importFileImportRecordJoin)
+   {
+      this.importFileImportRecordJoin = importFileImportRecordJoin;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importFileImportRecordJoinWidget
+    *******************************************************************************/
+   public QWidgetMetaDataInterface getImportFileImportRecordJoinWidget()
+   {
+      return (this.importFileImportRecordJoinWidget);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importFileImportRecordJoinWidget
+    *******************************************************************************/
+   public void setImportFileImportRecordJoinWidget(QWidgetMetaDataInterface importFileImportRecordJoinWidget)
+   {
+      this.importFileImportRecordJoinWidget = importFileImportRecordJoinWidget;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importFileImportRecordJoinWidget
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportFileImportRecordJoinWidget(QWidgetMetaDataInterface importFileImportRecordJoinWidget)
+   {
+      this.importFileImportRecordJoinWidget = importFileImportRecordJoinWidget;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importerProcessMetaDataBuilder
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder getImporterProcessMetaDataBuilder()
+   {
+      return (this.importerProcessMetaDataBuilder);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importerProcessMetaDataBuilder
+    *******************************************************************************/
+   public void setImporterProcessMetaDataBuilder(FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder)
+   {
+      this.importerProcessMetaDataBuilder = importerProcessMetaDataBuilder;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importerProcessMetaDataBuilder
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImporterProcessMetaDataBuilder(FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder)
+   {
+      this.importerProcessMetaDataBuilder = importerProcessMetaDataBuilder;
+      return (this);
+   }
+
+}
diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java
new file mode 100644
index 00000000..7e078f16
--- /dev/null
+++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java
@@ -0,0 +1,164 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.AbstractProcessMetaDataBuilder;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** Process MetaData Builder for FilesystemImporter process.
+ ** Meant to be used with (and actually is a parameter to the constructor of)
+ ** {@link FilesystemImporterMetaDataTemplate}
+ *******************************************************************************/
+public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMetaDataBuilder
+{
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder()
+   {
+      super(new QProcessMetaData()
+         .addStep(new QBackendStepMetaData()
+            .withName("sync")
+            .withCode(new QCodeReference(FilesystemImporterStep.class))
+            .withInputData(new QFunctionInputMetaData()
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_SOURCE_TABLE, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_FILE_FORMAT, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_FILE_TABLE, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_RECORD_TABLE, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_REMOVE_FILE_AFTER_IMPORT, QFieldType.BOOLEAN).withDefaultValue(true))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_UPDATE_FILE_IF_NAME_EXISTS, QFieldType.BOOLEAN).withDefaultValue(false))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, QFieldType.BOOLEAN).withDefaultValue(false))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_PATH, QFieldType.STRING))
+            )));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withSourceTableName(String sourceTableName)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_SOURCE_TABLE, sourceTableName);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withFileFormat(String fileFormat)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_FILE_FORMAT, fileFormat);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withImportFileTable(String importFileTable)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_FILE_TABLE, importFileTable);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withImportRecordTable(String importRecordTable)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_RECORD_TABLE, importRecordTable);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withRemoveFileAfterImport(boolean removeFileAfterImport)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_REMOVE_FILE_AFTER_IMPORT, removeFileAfterImport);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withUpdateFileIfNameExists(boolean updateFileIfNameExists)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_UPDATE_FILE_IF_NAME_EXISTS, updateFileIfNameExists);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withArchiveFileEnabled(boolean archiveFileEnabled)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, archiveFileEnabled);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withArchiveTableName(String archiveTableName)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, archiveTableName);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withArchivePath(String archivePath)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_ARCHIVE_PATH, archivePath);
+      return (this);
+   }
+
+}
diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java
new file mode 100644
index 00000000..c2b82495
--- /dev/null
+++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java
@@ -0,0 +1,363 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
+import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** BackendStep for FilesystemImporter process
+ **
+ ** Job is to:
+ ** - foreach file in the `source` table (e.g., a ONE-type filesystem table):
+ **   - optionally create an archive/backup copy of the file
+ **   - create a record in the `importFile` table
+ **   - parse the file, creating many records in the `importRecord` table
+ **   - remove the file from the `source` (if so configured (e.g., may turn off for Read-only FS))
+ *******************************************************************************/
+@SuppressWarnings("unchecked")
+public class FilesystemImporterStep implements BackendStep
+{
+   private static final QLogger LOG = QLogger.getLogger(FilesystemImporterStep.class);
+
+   public static final String FIELD_SOURCE_TABLE        = "sourceTable";
+   public static final String FIELD_FILE_FORMAT         = "fileFormat";
+   public static final String FIELD_IMPORT_FILE_TABLE   = "importFileTable";
+   public static final String FIELD_IMPORT_RECORD_TABLE = "importRecordTable";
+
+   public static final String FIELD_ARCHIVE_FILE_ENABLED     = "archiveFileEnabled";
+   public static final String FIELD_ARCHIVE_TABLE_NAME       = "archiveTableName";
+   public static final String FIELD_ARCHIVE_PATH             = "archivePath";
+   public static final String FIELD_REMOVE_FILE_AFTER_IMPORT = "removeFileAfterImport";
+
+   public static final String FIELD_UPDATE_FILE_IF_NAME_EXISTS = "updateFileIfNameExists";
+
+
+
+   /*******************************************************************************
+    ** Execute the step - using the request as input, and the result as output.
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // defer to a private method here, so we can add a type-parameter for that method to use              //
+      // would think we could do that here, but get compiler error, since this method comes from base class //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////
+      doRun(runBackendStepInput, runBackendStepOutput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <F> void doRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      String  fileFormat             = runBackendStepInput.getValueString(FIELD_FILE_FORMAT);
+      Boolean removeFileAfterImport  = runBackendStepInput.getValueBoolean(FIELD_REMOVE_FILE_AFTER_IMPORT);
+      Boolean updateFileIfNameExists = runBackendStepInput.getValueBoolean(FIELD_UPDATE_FILE_IF_NAME_EXISTS);
+      Boolean archiveFileEnabled     = runBackendStepInput.getValueBoolean(FIELD_ARCHIVE_FILE_ENABLED);
+
+      QTableMetaData sourceTable     = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
+      QTableMetaData importFileTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_IMPORT_FILE_TABLE));
+
+      String missingFieldErrorPrefix = "Process " + runBackendStepInput.getProcessName() + " was misconfigured - missing value in field: ";
+      Objects.requireNonNull(fileFormat, missingFieldErrorPrefix + FIELD_FILE_FORMAT);
+
+      ///////////////////////////////////////////////////////////////////////////////////
+      // list files in the backend system                                              //
+      // todo - can we do this using query action, with this being a "ONE" type table? //
+      ///////////////////////////////////////////////////////////////////////////////////
+      QBackendMetaData                    sourceBackend    = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
+      FilesystemBackendModuleInterface<F> sourceModule     = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
+      AbstractBaseFilesystemAction<F>     sourceActionBase = sourceModule.getActionBase();
+      sourceActionBase.preAction(sourceBackend);
+      Map<String, F> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
+
+      if(CollectionUtils.nullSafeIsEmpty(sourceFiles))
+      {
+         LOG.debug("No files found in import filesystem", logPair("sourceTable", sourceTable));
+         return;
+      }
+
+      ////////////////////////////////////////////////////////
+      // look up any existing file records with those names //
+      ////////////////////////////////////////////////////////
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(importFileTable.getName());
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("sourceFileName", QCriteriaOperator.IN, sourceFiles.keySet())));
+
+      QueryOutput               queryOutput           = new QueryAction().execute(queryInput);
+      Map<String, Serializable> existingImportedFiles = CollectionUtils.listToMap(queryOutput.getRecords(), r -> r.getValueString("sourceFileName"), r -> r.getValue("id"));
+
+      for(Map.Entry<String, F> sourceEntry : sourceFiles.entrySet())
+      {
+         QBackendTransaction transaction = null;
+         try
+         {
+            String sourceFileName = sourceEntry.getKey();
+
+            /////////////////////////////////////////////////////////
+            // if filename was already imported, decide what to do //
+            /////////////////////////////////////////////////////////
+            boolean      alreadyImported = existingImportedFiles.containsKey(sourceFileName);
+            Serializable idToUpdate      = null;
+            if(alreadyImported)
+            {
+               //////////////////////////////////////////////////////////////////////////////////
+               // todo - would we want to support importing multiple-times the same file name? //
+               // possibly - if so, add it here, presumably w/ another boolean field           //
+               //////////////////////////////////////////////////////////////////////////////////
+               if(updateFileIfNameExists)
+               {
+                  LOG.info("Updating already-imported file", logPair("fileName", sourceFileName), logPair("id", idToUpdate));
+                  idToUpdate = existingImportedFiles.get(sourceFileName);
+               }
+               else
+               {
+                  LOG.debug("Skipping already-imported file", logPair("fileName", sourceFileName));
+                  continue;
+               }
+            }
+
+            ///////////////////////////////////
+            // read the file as input stream //
+            ///////////////////////////////////
+            try(InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()))
+            {
+               byte[] bytes = inputStream.readAllBytes();
+
+               //////////////////////////////////////
+               // archive the file, if so directed //
+               //////////////////////////////////////
+               String archivedPath = null;
+               if(archiveFileEnabled)
+               {
+                  archivedPath = archiveFile(runBackendStepInput, sourceFileName, bytes);
+               }
+
+               /////////////////////////////////
+               // build record for importFile //
+               /////////////////////////////////
+               LOG.info("Syncing file [" + sourceFileName + "]");
+               QRecord importFileRecord = new QRecord()
+                  // todo - how to get clientId in here?
+                  .withValue("id", idToUpdate)
+                  .withValue("sourceFileName", sourceFileName)
+                  .withValue("archivedPath", archivedPath);
+
+               //////////////////////////////////////
+               // build child importRecord records //
+               //////////////////////////////////////
+               String content = new String(bytes);
+               importFileRecord.withAssociatedRecords("importRecords", parseFileIntoRecords(runBackendStepInput, content));
+
+               ///////////////////////////////////////////////////////////////////
+               // insert the file & records (records as association under file) //
+               ///////////////////////////////////////////////////////////////////
+               InsertAction insertAction = new InsertAction();
+               InsertInput  insertInput  = new InsertInput();
+               insertInput.setTableName(importFileTable.getName());
+               insertInput.setRecords(List.of(importFileRecord));
+
+               transaction = QBackendTransaction.openFor(insertInput);
+               insertInput.setTransaction(transaction);
+
+               InsertOutput insertOutput = insertAction.execute(insertInput);
+
+               LOG.info("Inserted insertFile & records", logPair("id", insertOutput.getRecords().get(0).getValue("id")));
+
+               transaction.commit();
+            }
+
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            // after the records are built, we can delete the file                                       //
+            // if we are interrupted between the commit & the delete, then the file will be found again, //
+            // and we'll either skip it or do an update, based on FIELD_UPDATE_FILE_IF_NAME_EXISTS flag  //
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            if(removeFileAfterImport)
+            {
+               String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
+               sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.error("Error processing file: " + sourceEntry, e);
+            if(transaction != null)
+            {
+               transaction.rollback();
+            }
+         }
+         finally
+         {
+            if(transaction != null)
+            {
+               transaction.close();
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String archiveFile(RunBackendStepInput runBackendStepInput, String sourceFileName, byte[] bytes) throws QException, IOException
+   {
+      String         archiveTableName = runBackendStepInput.getValueString(FIELD_ARCHIVE_TABLE_NAME);
+      QTableMetaData archiveTable;
+      try
+      {
+         archiveTable = runBackendStepInput.getInstance().getTable(archiveTableName);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error getting archive table [" + archiveTableName + "]", e));
+      }
+
+      String archivePath = Objects.requireNonNullElse(runBackendStepInput.getValueString(FIELD_ARCHIVE_PATH), "");
+
+      QBackendMetaData                    archiveBackend    = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
+      FilesystemBackendModuleInterface<?> archiveModule     = (FilesystemBackendModuleInterface<?>) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
+      AbstractBaseFilesystemAction<?>     archiveActionBase = archiveModule.getActionBase();
+      archiveActionBase.preAction(archiveBackend);
+
+      LocalDateTime now = LocalDateTime.now();
+      String path = archiveActionBase.getFullBasePath(archiveTable, archiveBackend)
+         + File.separator + archivePath
+         + File.separator + now.getYear()
+         + File.separator + now.getMonth()
+         + File.separator + UUID.randomUUID()
+         + "-" + sourceFileName.replaceAll(".*" + File.separator, "");
+      LOG.info("Archiving file", logPair("path", path));
+      archiveActionBase.writeFile(archiveBackend, path, bytes);
+
+      return (path);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:Indentation")
+   List<QRecord> parseFileIntoRecords(RunBackendStepInput runBackendStepInput, String content) throws QException
+   {
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first, parse the content into records, w/ unknown field names - just whatever is in the CSV or JSON //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      String fileFormat = runBackendStepInput.getValueString(FIELD_FILE_FORMAT);
+
+      List<QRecord> contentRecords = switch(fileFormat.toLowerCase())
+      {
+         case "csv" ->
+         {
+            CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+            csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
+               .withCsv(content)
+               .withCaseSensitiveHeaders(true)
+               .withCsvHeadersAsFieldNames(true)
+            );
+            yield (csvToQRecordAdapter.getRecordList());
+         }
+
+         case "json" -> new JsonToQRecordAdapter().buildRecordsFromJson(content, null, null);
+
+         default -> throw (new QException("Unexpected file format: " + fileFormat));
+      };
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now, wrap those records with the fields of the importRecord table, putting the unknown fields in a blob together //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<QRecord> importRecordList = new ArrayList<>();
+      int           recordNo         = 1;
+      for(QRecord record : contentRecords)
+      {
+         record.setValue("recordNo", recordNo++);
+         // todo - client_id??
+
+         importRecordList.add(record);
+      }
+
+      return (importRecordList);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
+   {
+      List<F>        files = actionBase.listFiles(table, backend);
+      Map<String, F> rs    = new LinkedHashMap<>();
+
+      for(F file : files)
+      {
+         String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
+         rs.put(fileName, file);
+      }
+
+      return (rs);
+   }
+
+}
diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizer.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizer.java
new file mode 100644
index 00000000..eff40b93
--- /dev/null
+++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizer.java
@@ -0,0 +1,82 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** combine all unstructured fields of the record into a JSON blob in the "values" field.
+ *******************************************************************************/
+public class ImportRecordPostQueryCustomizer extends AbstractPostQueryCustomizer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> apply(List<QRecord> records)
+   {
+      if(CollectionUtils.nullSafeHasContents(records))
+      {
+         QTableMetaData table = null;
+         if(StringUtils.hasContent(records.get(0).getTableName()))
+         {
+            table = QContext.getQInstance().getTable(records.get(0).getTableName());
+         }
+
+         for(QRecord record : records)
+         {
+            Map<String, Serializable> values = record.getValues();
+
+            if(table != null)
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // remove known values from a clone of the values map - then only put the un-structured values in a JSON document in the values field //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               values = new HashMap<>(values);
+               for(String fieldName : table.getFields().keySet())
+               {
+                  values.remove(fieldName);
+               }
+            }
+
+            String valuesJson = JsonUtils.toJson(values);
+            record.setValue("values", valuesJson);
+         }
+      }
+
+      return (records);
+   }
+
+}
diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
index cab98928..7696c6b3 100644
--- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
+++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -38,12 +39,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed.StreamedETLFilesystemBackendStep;
+import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer.FilesystemImporterMetaDataTemplate;
+import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer.FilesystemImporterProcessMetaDataBuilder;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
@@ -59,16 +63,19 @@ public class TestUtils
    public static final String BACKEND_NAME_S3             = "s3";
    public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
    public static final String BACKEND_NAME_MOCK           = "mock";
+   public static final String BACKEND_NAME_MEMORY = "memory";
 
    public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
    public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV  = "person-local-csv";
    public static final String TABLE_NAME_BLOB_LOCAL_FS        = "local-blob";
+   public static final String TABLE_NAME_ARCHIVE_LOCAL_FS    = "local-archive";
    public static final String TABLE_NAME_PERSON_S3            = "person-s3";
    public static final String TABLE_NAME_BLOB_S3              = "s3-blob";
    public static final String TABLE_NAME_PERSON_MOCK          = "person-mock";
    public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix";
 
-   public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
+   public static final String PROCESS_NAME_STREAMED_ETL                   = "etl.streamed";
+   public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter";
 
    ///////////////////////////////////////////////////////////////////
    // shouldn't be accessed directly, as we append a counter to it. //
@@ -136,15 +143,30 @@ public class TestUtils
       qInstance.addTable(defineLocalFilesystemJSONPersonTable());
       qInstance.addTable(defineLocalFilesystemCSVPersonTable());
       qInstance.addTable(defineLocalFilesystemBlobTable());
+      qInstance.addTable(defineLocalFilesystemArchiveTable());
       qInstance.addBackend(defineS3Backend());
       qInstance.addBackend(defineS3BackendSansPrefix());
       qInstance.addTable(defineS3CSVPersonTable());
       qInstance.addTable(defineS3BlobTable());
       qInstance.addTable(defineS3BlobSansPrefixTable());
       qInstance.addBackend(defineMockBackend());
+      qInstance.addBackend(defineMemoryBackend());
       qInstance.addTable(defineMockPersonTable());
       qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
 
+      String importBaseName = "personImporter";
+      FilesystemImporterProcessMetaDataBuilder filesystemImporterProcessMetaDataBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
+         .withSourceTableName(TABLE_NAME_PERSON_LOCAL_FS_CSV)
+         .withFileFormat("csv")
+         .withArchiveFileEnabled(true)
+         .withArchiveTableName(TABLE_NAME_ARCHIVE_LOCAL_FS)
+         .withArchivePath("archive-of/personImporterFiles")
+         .withName(LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+
+      FilesystemImporterMetaDataTemplate filesystemImporterMetaDataTemplate = new FilesystemImporterMetaDataTemplate(qInstance, importBaseName, BACKEND_NAME_MEMORY, filesystemImporterProcessMetaDataBuilder, table -> table.withAuditRules(QAuditRules.defaultInstanceLevelNone()));
+
+      filesystemImporterMetaDataTemplate.addToInstance(qInstance);
+
       return (qInstance);
    }
 
@@ -257,6 +279,28 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineLocalFilesystemArchiveTable()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_ARCHIVE_LOCAL_FS)
+         .withLabel("Archive")
+         .withBackendName(defineLocalFilesystemBackend().getName())
+         .withPrimaryKeyField("fileName")
+         .withField(new QFieldMetaData("fileName", QFieldType.STRING))
+         .withField(new QFieldMetaData("contents", QFieldType.BLOB))
+         .withBackendDetails(new FilesystemTableBackendDetails()
+            .withBasePath("archive")
+            .withCardinality(Cardinality.ONE)
+            .withFileNameFieldName("fileName")
+            .withContentsFieldName("contents")
+         );
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -356,6 +400,18 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QBackendMetaData defineMemoryBackend()
+   {
+      return (new QBackendMetaData()
+         .withBackendType(MemoryBackendModule.class)
+         .withName(BACKEND_NAME_MEMORY));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java
new file mode 100644
index 00000000..794e0e15
--- /dev/null
+++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java
@@ -0,0 +1,114 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.io.File;
+import java.time.LocalDateTime;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import org.json.JSONObject;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+
+/*******************************************************************************
+ ** Unit test for FilesystemImporterStep 
+ *******************************************************************************/
+class FilesystemImporterStepTest extends FilesystemActionTest
+{
+
+   //////////////////////////////////////////////////////////////////////////
+   // note - we take advantage of the @BeforeEach and @AfterEach to set up //
+   // and clean up files on disk for this test.                            //
+   //////////////////////////////////////////////////////////////////////////
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   public void filesystemBaseAfterEach() throws Exception
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      String importBaseName = "personImporter";
+      assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+
+      QRecord record = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(1, record.getValue("importFileId"));
+      assertEquals("John", record.getValue("firstName"));
+      assertThat(record.getValue("values")).isInstanceOf(String.class);
+      JSONObject values = new JSONObject(record.getValueString("values"));
+      assertEquals("John", values.get("firstName"));
+
+      FilesystemBackendMetaData backend  = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
+      String                    basePath = backend.getBasePath();
+      System.out.println(basePath);
+
+      ///////////////////////////////////////////
+      // make sure 2 archive files got created //
+      ///////////////////////////////////////////
+      LocalDateTime now   = LocalDateTime.now();
+      File[]        files = new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth()).listFiles();
+      assertNotNull(files);
+      assertEquals(2, files.length);
+   }
+
+   // todo - test json
+
+   // todo - test no files found
+
+   // todo - confirm delete happens?
+
+   // todo - updates?
+
+}
\ No newline at end of file
diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizerTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizerTest.java
new file mode 100644
index 00000000..957078df
--- /dev/null
+++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizerTest.java
@@ -0,0 +1,85 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
+import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
+import org.json.JSONObject;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for ImportRecordPostQueryCustomizer 
+ *******************************************************************************/
+class ImportRecordPostQueryCustomizerTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      Instant createDate = Instant.parse("2024-01-08T20:07:21Z");
+
+      List<QRecord> output = new ImportRecordPostQueryCustomizer().apply(List.of(
+         new QRecord()
+            .withTableName("personImporterImportRecord")
+            .withValue("importFileId", 1)
+            .withValue("unmapped", 2)
+            .withValue("unstructured", 3)
+            .withValue("nosqlObject", MapBuilder.of(HashMap::new).with("foo", "bar").with("createDate", createDate).build())
+      ));
+
+      assertEquals(1, output.get(0).getValue("importFileId"));
+      assertEquals(2, output.get(0).getValue("unmapped"));
+      assertEquals(3, output.get(0).getValue("unstructured"));
+      assertEquals(Map.of("foo", "bar", "createDate", createDate), output.get(0).getValue("nosqlObject"));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // make sure all un-structured fields get put in the "values" field as a JSON string     //
+      // compare as maps, beacuse JSONObject seems to care about the ordering, which, we don't //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      Map<String, Object> expectedMap = new JSONObject("""
+         {
+            "unmapped": 2,
+            "unstructured": 3,
+            "nosqlObject":
+            {
+               "foo": "bar",
+               "createDate": "%s"
+            }
+         }
+         """.formatted(createDate)).toMap();
+      Map<String, Object> actualMap = new JSONObject(output.get(0).getValueString("values")).toMap();
+      assertThat(actualMap).isEqualTo(expectedMap);
+   }
+
+}
\ No newline at end of file