QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-781 Initial checkin of filesystem importer meta-data template and process

Browse Source
This commit is contained in:
darin.kelkhoff
2024-01-11 07:46:30 -06:00
parent b64883f34f
commit 624a723b54
8 changed files with 1566 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java Normal file
View File

@ -0,0 +1,199 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemTableMetaDataBuilder
{
private String name;
private QBackendMetaData backend;
private String basePath;
private String glob;
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("checkstyle:Indentation")
public QTableMetaData buildStandardCardinalityOneTable()
{
AbstractFilesystemTableBackendDetails tableBackendDetails = switch(backend.getBackendType())
{
case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails();
case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails();
default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType());
};
return new QTableMetaData()
.withName(name)
.withIsHidden(true)
.withBackendName(backend.getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.INTEGER))
.withField(new QFieldMetaData("contents", QFieldType.STRING))
.withBackendDetails(tableBackendDetails
.withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")
.withContentsFieldName("contents")
.withBasePath(basePath)
.withGlob(glob));
}
/*******************************************************************************
** Getter for backend
*******************************************************************************/
public QBackendMetaData getBackend()
{
return (this.backend);
}
/*******************************************************************************
** Setter for backend
*******************************************************************************/
public void setBackend(QBackendMetaData backend)
{
this.backend = backend;
}
/*******************************************************************************
** Fluent setter for backend
*******************************************************************************/
public FilesystemTableMetaDataBuilder withBackend(QBackendMetaData backend)
{
this.backend = backend;
return (this);
}
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getName()
{
return (this.name);
}
/*******************************************************************************
** Setter for tableName
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
*******************************************************************************/
public FilesystemTableMetaDataBuilder withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for basePath
*******************************************************************************/
public String getBasePath()
{
return (this.basePath);
}
/*******************************************************************************
** Setter for basePath
*******************************************************************************/
public void setBasePath(String basePath)
{
this.basePath = basePath;
}
/*******************************************************************************
** Fluent setter for basePath
*******************************************************************************/
public FilesystemTableMetaDataBuilder withBasePath(String basePath)
{
this.basePath = basePath;
return (this);
}
/*******************************************************************************
** Getter for glob
*******************************************************************************/
public String getGlob()
{
return (this.glob);
}
/*******************************************************************************
** Setter for glob
*******************************************************************************/
public void setGlob(String glob)
{
this.glob = glob;
}
/*******************************************************************************
** Fluent setter for glob
*******************************************************************************/
public FilesystemTableMetaDataBuilder withGlob(String glob)
{
this.glob = glob;
return (this);
}
}

502
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java Normal file
View File

@ -0,0 +1,502 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
import java.util.List;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
/*******************************************************************************
** Class to serve as a template for producing an instance of a process & tables
** that provide the QQQ service to manage importing files (e.g., partner feeds on S3).
**
** The template contains the following components:
** - A process that loads files from a source-table (e.g., of filesystem, cardinality=ONE)
** and stores them in the following tables:
** - {baseName}importFile table - simple header for imported files.
** - {baseName}importRecord table - a record foreach record in an imported file.
** - PVS for the importFile table
** - Join & Widget (to show importRecords on importFile view screen)
**
** Most likely one would add all the meta-data objects in an instance of this
** template, then either use tableAutomations or a basepull process against records
** in the importRecord table, to run through a process (e.g., an AbstractTableSync)
** to result in final values for your business case.
**
** A typical usage may look like:
**
** <pre>
// set up the process that'll be used to import the files.
FilesystemImporterProcessMetaDataBuilder importerProcessBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
.withFileFormat("csv")
.withSourceTableName(MyFeedSourceTableMetaDataProducer.NAME)
.withRemoveFileAfterImport(true)
.withUpdateFileIfNameExists(false)
.withName("myFeedImporter")
.withSchedule(new QScheduleMetaData().withRepeatSeconds(300));
FilesystemImporterMetaDataTemplate template = new FilesystemImporterMetaDataTemplate(qInstance, "myFeed", MongoDBMetaDataProducer.NAME, importerProcessBuilder, table ->
{
// whatever customizations you may need on the tables
table.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED));
});
// set up automations on the table
template.addAutomationStatusField(template.getImportRecordTable(), getStandardAutomationStatusField().withBackendName("metaData.automationStatus"));
template.addStandardPostInsertAutomation(template.getImportRecordTable(), getBasicTableAutomationDetails(), "myFeedTableSyncProcess");
// finally, add all the meta-data from the template to a QInstance
template.addToInstance(qInstance);
<pre>
**
*******************************************************************************/
public class FilesystemImporterMetaDataTemplate
{
public static final String IMPORT_FILE_TABLE_SUFFIX = "ImportFile";
public static final String IMPORT_RECORD_TABLE_SUFFIX = "ImportRecord";
public static final String IMPORT_FILE_RECORD_JOIN_SUFFIX = "ImportFileImportRecordJoin";
private QTableMetaData importFileTable;
private QTableMetaData importRecordTable;
private QPossibleValueSource importFilePVS;
private QJoinMetaData importFileImportRecordJoin;
private QWidgetMetaDataInterface importFileImportRecordJoinWidget;
private FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder;
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterMetaDataTemplate(QInstance qInstance, String importBaseName, String backendName, FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder, Consumer<QTableMetaData> tableEnricher)
{
QBackendMetaData backend = qInstance.getBackend(backendName);
this.importFileTable = defineTableImportFile(backend, importBaseName);
this.importRecordTable = defineTableImportRecord(backend, importBaseName);
for(QTableMetaData table : List.of(this.importFileTable, this.importRecordTable))
{
table.setBackendName(backendName);
if(tableEnricher != null)
{
tableEnricher.accept(table);
}
}
this.importFilePVS = QPossibleValueSource.newForTable(this.importFileTable.getName());
this.importFileImportRecordJoin = defineImportFileImportRecordJoin(importBaseName);
this.importFileImportRecordJoinWidget = defineImportFileImportRecordChildWidget(this.importFileImportRecordJoin);
this.importerProcessMetaDataBuilder = importerProcessMetaDataBuilder
.withImportFileTable(this.importFileTable.getName())
.withImportRecordTable(this.importRecordTable.getName());
}
/*******************************************************************************
**
*******************************************************************************/
public void addAutomationStatusField(QTableMetaData table, QFieldMetaData automationStatusField)
{
table.addField(automationStatusField);
table.getSections().get(1).getFieldNames().add(0, automationStatusField.getName());
}
/*******************************************************************************
**
*******************************************************************************/
public TableAutomationAction addStandardPostInsertAutomation(QTableMetaData table, QTableAutomationDetails automationDetails, String processName)
{
TableAutomationAction action = new TableAutomationAction()
.withName(table.getName() + "PostInsert")
.withTriggerEvent(TriggerEvent.POST_INSERT)
.withProcessName(processName);
table.withAutomationDetails(automationDetails
.withAction(action));
return (action);
}
/*******************************************************************************
**
*******************************************************************************/
public QWidgetMetaDataInterface defineImportFileImportRecordChildWidget(QJoinMetaData join)
{
return ChildRecordListRenderer.widgetMetaDataBuilder(join)
.withName(join.getName())
.withLabel("Import Records")
.withCanAddChildRecord(false)
.getWidgetMetaData();
}
/*******************************************************************************
**
*******************************************************************************/
public QJoinMetaData defineImportFileImportRecordJoin(String importBaseName)
{
return new QJoinMetaData()
.withLeftTable(importBaseName + IMPORT_FILE_TABLE_SUFFIX)
.withRightTable(importBaseName + IMPORT_RECORD_TABLE_SUFFIX)
.withName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX)
.withType(JoinType.ONE_TO_MANY)
.withJoinOn(new JoinOn("id", "importFileId"));
}
/*******************************************************************************
**
*******************************************************************************/
public QTableMetaData defineTableImportFile(QBackendMetaData backend, String importBaseName)
{
QFieldType idType = getIdFieldType(backend);
QTableMetaData qTableMetaData = new QTableMetaData()
.withName(importBaseName + IMPORT_FILE_TABLE_SUFFIX)
.withIcon(new QIcon().withName("upload_file"))
.withRecordLabelFormat("%s")
.withRecordLabelFields("sourceFileName")
.withPrimaryKeyField("id")
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD))
.withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend)))
.withField(new QFieldMetaData("sourceFileName", QFieldType.STRING))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName")))
.withSection(new QFieldSection("records", new QIcon().withName("power_input"), Tier.T2).withWidgetName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX))
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
.withAssociation(new Association().withName("importRecords").withJoinName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX).withAssociatedTableName(importBaseName + IMPORT_RECORD_TABLE_SUFFIX));
return (qTableMetaData);
}
/*******************************************************************************
**
*******************************************************************************/
public QFieldType getIdFieldType(QBackendMetaData backend)
{
QFieldType idType = QFieldType.INTEGER;
if("mongodb".equals(backend.getBackendType()))
{
idType = QFieldType.STRING;
}
return idType;
}
/*******************************************************************************
**
*******************************************************************************/
public String getIdFieldBackendName(QBackendMetaData backend)
{
if("mongodb".equals(backend.getBackendType()))
{
return ("_id");
}
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
public QTableMetaData defineTableImportRecord(QBackendMetaData backend, String importBaseName)
{
QFieldType idType = getIdFieldType(backend);
QTableMetaData qTableMetaData = new QTableMetaData()
.withName(importBaseName + IMPORT_RECORD_TABLE_SUFFIX)
.withIcon(new QIcon().withName("power_input"))
.withRecordLabelFormat("%s")
.withRecordLabelFields("importFileId", "recordNo")
.withPrimaryKeyField("id")
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD))
.withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(ImportRecordPostQueryCustomizer.class))
.withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend)))
.withField(new QFieldMetaData("importFileId", idType).withBackendName("metaData.importFileId")
.withPossibleValueSourceName(importBaseName + IMPORT_FILE_TABLE_SUFFIX))
.withField(new QFieldMetaData("recordNo", QFieldType.INTEGER).withBackendName("metaData.recordNo"))
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// so, we'll use this field as a "virtual" field, e.g., populated with JSON in table post-query customizer, with all un-structured values //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
.withField(new QFieldMetaData("values", QFieldType.TEXT)
.withIsEditable(false)
.withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR)
.withValue(AdornmentType.CodeEditorValues.languageMode("json"))))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("metaData.createDate").withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("metaData.modifyDate").withIsEditable(false))
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "importFileId", "recordNo")))
.withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("values")))
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
return (qTableMetaData);
}
/*******************************************************************************
**
*******************************************************************************/
public void addToInstance(QInstance instance)
{
instance.add(importFileTable);
instance.add(importRecordTable);
instance.add(importFilePVS);
instance.add(importFileImportRecordJoin);
instance.add(importFileImportRecordJoinWidget);
instance.add(importerProcessMetaDataBuilder.getProcessMetaData());
}
/*******************************************************************************
** Getter for importFileTable
*******************************************************************************/
public QTableMetaData getImportFileTable()
{
return (this.importFileTable);
}
/*******************************************************************************
** Setter for importFileTable
*******************************************************************************/
public void setImportFileTable(QTableMetaData importFileTable)
{
this.importFileTable = importFileTable;
}
/*******************************************************************************
** Fluent setter for importFileTable
*******************************************************************************/
public FilesystemImporterMetaDataTemplate withImportFileTable(QTableMetaData importFileTable)
{
this.importFileTable = importFileTable;
return (this);
}
/*******************************************************************************
** Getter for importRecordTable
*******************************************************************************/
public QTableMetaData getImportRecordTable()
{
return (this.importRecordTable);
}
/*******************************************************************************
** Setter for importRecordTable
*******************************************************************************/
public void setImportRecordTable(QTableMetaData importRecordTable)
{
this.importRecordTable = importRecordTable;
}
/*******************************************************************************
** Fluent setter for importRecordTable
*******************************************************************************/
public FilesystemImporterMetaDataTemplate withImportRecordTable(QTableMetaData importRecordTable)
{
this.importRecordTable = importRecordTable;
return (this);
}
/*******************************************************************************
** Getter for importFilePVS
*******************************************************************************/
public QPossibleValueSource getImportFilePVS()
{
return (this.importFilePVS);
}
/*******************************************************************************
** Setter for importFilePVS
*******************************************************************************/
public void setImportFilePVS(QPossibleValueSource importFilePVS)
{
this.importFilePVS = importFilePVS;
}
/*******************************************************************************
** Fluent setter for importFilePVS
*******************************************************************************/
public FilesystemImporterMetaDataTemplate withImportFilePVS(QPossibleValueSource importFilePVS)
{
this.importFilePVS = importFilePVS;
return (this);
}
/*******************************************************************************
** Getter for importFileImportRecordJoin
*******************************************************************************/
public QJoinMetaData getImportFileImportRecordJoin()
{
return (this.importFileImportRecordJoin);
}
/*******************************************************************************
** Setter for importFileImportRecordJoin
*******************************************************************************/
public void setImportFileImportRecordJoin(QJoinMetaData importFileImportRecordJoin)
{
this.importFileImportRecordJoin = importFileImportRecordJoin;
}
/*******************************************************************************
** Fluent setter for importFileImportRecordJoin
*******************************************************************************/
public FilesystemImporterMetaDataTemplate withImportFileImportRecordJoin(QJoinMetaData importFileImportRecordJoin)
{
this.importFileImportRecordJoin = importFileImportRecordJoin;
return (this);
}
/*******************************************************************************
** Getter for importFileImportRecordJoinWidget
*******************************************************************************/
public QWidgetMetaDataInterface getImportFileImportRecordJoinWidget()
{
return (this.importFileImportRecordJoinWidget);
}
/*******************************************************************************
** Setter for importFileImportRecordJoinWidget
*******************************************************************************/
public void setImportFileImportRecordJoinWidget(QWidgetMetaDataInterface importFileImportRecordJoinWidget)
{
this.importFileImportRecordJoinWidget = importFileImportRecordJoinWidget;
}
/*******************************************************************************
** Fluent setter for importFileImportRecordJoinWidget
*******************************************************************************/
public FilesystemImporterMetaDataTemplate withImportFileImportRecordJoinWidget(QWidgetMetaDataInterface importFileImportRecordJoinWidget)
{
this.importFileImportRecordJoinWidget = importFileImportRecordJoinWidget;
return (this);
}
/*******************************************************************************
** Getter for importerProcessMetaDataBuilder
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder getImporterProcessMetaDataBuilder()
{
return (this.importerProcessMetaDataBuilder);
}
/*******************************************************************************
** Setter for importerProcessMetaDataBuilder
*******************************************************************************/
public void setImporterProcessMetaDataBuilder(FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder)
{
this.importerProcessMetaDataBuilder = importerProcessMetaDataBuilder;
}
/*******************************************************************************
** Fluent setter for importerProcessMetaDataBuilder
*******************************************************************************/
public FilesystemImporterMetaDataTemplate withImporterProcessMetaDataBuilder(FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder)
{
this.importerProcessMetaDataBuilder = importerProcessMetaDataBuilder;
return (this);
}
}

164
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java Normal file
View File

@ -0,0 +1,164 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.AbstractProcessMetaDataBuilder;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
/*******************************************************************************
** Process MetaData Builder for FilesystemImporter process.
** Meant to be used with (and actually is a parameter to the constructor of)
** {@link FilesystemImporterMetaDataTemplate}
*******************************************************************************/
public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMetaDataBuilder
{
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder()
{
super(new QProcessMetaData()
.addStep(new QBackendStepMetaData()
.withName("sync")
.withCode(new QCodeReference(FilesystemImporterStep.class))
.withInputData(new QFunctionInputMetaData()
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_SOURCE_TABLE, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_FILE_FORMAT, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_FILE_TABLE, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_RECORD_TABLE, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_REMOVE_FILE_AFTER_IMPORT, QFieldType.BOOLEAN).withDefaultValue(true))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_UPDATE_FILE_IF_NAME_EXISTS, QFieldType.BOOLEAN).withDefaultValue(false))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, QFieldType.BOOLEAN).withDefaultValue(false))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_PATH, QFieldType.STRING))
)));
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withSourceTableName(String sourceTableName)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_SOURCE_TABLE, sourceTableName);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withFileFormat(String fileFormat)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_FILE_FORMAT, fileFormat);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withImportFileTable(String importFileTable)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_FILE_TABLE, importFileTable);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withImportRecordTable(String importRecordTable)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_RECORD_TABLE, importRecordTable);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withRemoveFileAfterImport(boolean removeFileAfterImport)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_REMOVE_FILE_AFTER_IMPORT, removeFileAfterImport);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withUpdateFileIfNameExists(boolean updateFileIfNameExists)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_UPDATE_FILE_IF_NAME_EXISTS, updateFileIfNameExists);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withArchiveFileEnabled(boolean archiveFileEnabled)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, archiveFileEnabled);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withArchiveTableName(String archiveTableName)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, archiveTableName);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withArchivePath(String archivePath)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_ARCHIVE_PATH, archivePath);
return (this);
}
}

363
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java Normal file
View File

@ -0,0 +1,363 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** BackendStep for FilesystemImporter process
**
** Job is to:
** - foreach file in the `source` table (e.g., a ONE-type filesystem table):
** - optionally create an archive/backup copy of the file
** - create a record in the `importFile` table
** - parse the file, creating many records in the `importRecord` table
** - remove the file from the `source` (if so configured (e.g., may turn off for Read-only FS))
*******************************************************************************/
@SuppressWarnings("unchecked")
public class FilesystemImporterStep implements BackendStep
{
private static final QLogger LOG = QLogger.getLogger(FilesystemImporterStep.class);
public static final String FIELD_SOURCE_TABLE = "sourceTable";
public static final String FIELD_FILE_FORMAT = "fileFormat";
public static final String FIELD_IMPORT_FILE_TABLE = "importFileTable";
public static final String FIELD_IMPORT_RECORD_TABLE = "importRecordTable";
public static final String FIELD_ARCHIVE_FILE_ENABLED = "archiveFileEnabled";
public static final String FIELD_ARCHIVE_TABLE_NAME = "archiveTableName";
public static final String FIELD_ARCHIVE_PATH = "archivePath";
public static final String FIELD_REMOVE_FILE_AFTER_IMPORT = "removeFileAfterImport";
public static final String FIELD_UPDATE_FILE_IF_NAME_EXISTS = "updateFileIfNameExists";
/*******************************************************************************
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// defer to a private method here, so we can add a type-parameter for that method to use //
// would think we could do that here, but get compiler error, since this method comes from base class //
////////////////////////////////////////////////////////////////////////////////////////////////////////
doRun(runBackendStepInput, runBackendStepOutput);
}
/*******************************************************************************
**
*******************************************************************************/
private <F> void doRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
String fileFormat = runBackendStepInput.getValueString(FIELD_FILE_FORMAT);
Boolean removeFileAfterImport = runBackendStepInput.getValueBoolean(FIELD_REMOVE_FILE_AFTER_IMPORT);
Boolean updateFileIfNameExists = runBackendStepInput.getValueBoolean(FIELD_UPDATE_FILE_IF_NAME_EXISTS);
Boolean archiveFileEnabled = runBackendStepInput.getValueBoolean(FIELD_ARCHIVE_FILE_ENABLED);
QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
QTableMetaData importFileTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_IMPORT_FILE_TABLE));
String missingFieldErrorPrefix = "Process " + runBackendStepInput.getProcessName() + " was misconfigured - missing value in field: ";
Objects.requireNonNull(fileFormat, missingFieldErrorPrefix + FIELD_FILE_FORMAT);
///////////////////////////////////////////////////////////////////////////////////
// list files in the backend system //
// todo - can we do this using query action, with this being a "ONE" type table? //
///////////////////////////////////////////////////////////////////////////////////
QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface<F> sourceModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
AbstractBaseFilesystemAction<F> sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend);
Map<String, F> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
if(CollectionUtils.nullSafeIsEmpty(sourceFiles))
{
LOG.debug("No files found in import filesystem", logPair("sourceTable", sourceTable));
return;
}
////////////////////////////////////////////////////////
// look up any existing file records with those names //
////////////////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(importFileTable.getName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("sourceFileName", QCriteriaOperator.IN, sourceFiles.keySet())));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Map<String, Serializable> existingImportedFiles = CollectionUtils.listToMap(queryOutput.getRecords(), r -> r.getValueString("sourceFileName"), r -> r.getValue("id"));
for(Map.Entry<String, F> sourceEntry : sourceFiles.entrySet())
{
QBackendTransaction transaction = null;
try
{
String sourceFileName = sourceEntry.getKey();
/////////////////////////////////////////////////////////
// if filename was already imported, decide what to do //
/////////////////////////////////////////////////////////
boolean alreadyImported = existingImportedFiles.containsKey(sourceFileName);
Serializable idToUpdate = null;
if(alreadyImported)
{
//////////////////////////////////////////////////////////////////////////////////
// todo - would we want to support importing multiple-times the same file name? //
// possibly - if so, add it here, presumably w/ another boolean field //
//////////////////////////////////////////////////////////////////////////////////
if(updateFileIfNameExists)
{
LOG.info("Updating already-imported file", logPair("fileName", sourceFileName), logPair("id", idToUpdate));
idToUpdate = existingImportedFiles.get(sourceFileName);
}
else
{
LOG.debug("Skipping already-imported file", logPair("fileName", sourceFileName));
continue;
}
}
///////////////////////////////////
// read the file as input stream //
///////////////////////////////////
try(InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()))
{
byte[] bytes = inputStream.readAllBytes();
//////////////////////////////////////
// archive the file, if so directed //
//////////////////////////////////////
String archivedPath = null;
if(archiveFileEnabled)
{
archivedPath = archiveFile(runBackendStepInput, sourceFileName, bytes);
}
/////////////////////////////////
// build record for importFile //
/////////////////////////////////
LOG.info("Syncing file [" + sourceFileName + "]");
QRecord importFileRecord = new QRecord()
// todo - how to get clientId in here?
.withValue("id", idToUpdate)
.withValue("sourceFileName", sourceFileName)
.withValue("archivedPath", archivedPath);
//////////////////////////////////////
// build child importRecord records //
//////////////////////////////////////
String content = new String(bytes);
importFileRecord.withAssociatedRecords("importRecords", parseFileIntoRecords(runBackendStepInput, content));
///////////////////////////////////////////////////////////////////
// insert the file & records (records as association under file) //
///////////////////////////////////////////////////////////////////
InsertAction insertAction = new InsertAction();
InsertInput insertInput = new InsertInput();
insertInput.setTableName(importFileTable.getName());
insertInput.setRecords(List.of(importFileRecord));
transaction = QBackendTransaction.openFor(insertInput);
insertInput.setTransaction(transaction);
InsertOutput insertOutput = insertAction.execute(insertInput);
LOG.info("Inserted insertFile & records", logPair("id", insertOutput.getRecords().get(0).getValue("id")));
transaction.commit();
}
///////////////////////////////////////////////////////////////////////////////////////////////
// after the records are built, we can delete the file //
// if we are interrupted between the commit & the delete, then the file will be found again, //
// and we'll either skip it or do an update, based on FIELD_UPDATE_FILE_IF_NAME_EXISTS flag //
///////////////////////////////////////////////////////////////////////////////////////////////
if(removeFileAfterImport)
{
String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
}
}
catch(Exception e)
{
LOG.error("Error processing file: " + sourceEntry, e);
if(transaction != null)
{
transaction.rollback();
}
}
finally
{
if(transaction != null)
{
transaction.close();
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private String archiveFile(RunBackendStepInput runBackendStepInput, String sourceFileName, byte[] bytes) throws QException, IOException
{
String archiveTableName = runBackendStepInput.getValueString(FIELD_ARCHIVE_TABLE_NAME);
QTableMetaData archiveTable;
try
{
archiveTable = runBackendStepInput.getInstance().getTable(archiveTableName);
}
catch(Exception e)
{
throw (new QException("Error getting archive table [" + archiveTableName + "]", e));
}
String archivePath = Objects.requireNonNullElse(runBackendStepInput.getValueString(FIELD_ARCHIVE_PATH), "");
QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface<?> archiveModule = (FilesystemBackendModuleInterface<?>) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
AbstractBaseFilesystemAction<?> archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend);
LocalDateTime now = LocalDateTime.now();
String path = archiveActionBase.getFullBasePath(archiveTable, archiveBackend)
+ File.separator + archivePath
+ File.separator + now.getYear()
+ File.separator + now.getMonth()
+ File.separator + UUID.randomUUID()
+ "-" + sourceFileName.replaceAll(".*" + File.separator, "");
LOG.info("Archiving file", logPair("path", path));
archiveActionBase.writeFile(archiveBackend, path, bytes);
return (path);
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("checkstyle:Indentation")
List<QRecord> parseFileIntoRecords(RunBackendStepInput runBackendStepInput, String content) throws QException
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// first, parse the content into records, w/ unknown field names - just whatever is in the CSV or JSON //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
String fileFormat = runBackendStepInput.getValueString(FIELD_FILE_FORMAT);
List<QRecord> contentRecords = switch(fileFormat.toLowerCase())
{
case "csv" ->
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
.withCsv(content)
.withCaseSensitiveHeaders(true)
.withCsvHeadersAsFieldNames(true)
);
yield (csvToQRecordAdapter.getRecordList());
}
case "json" -> new JsonToQRecordAdapter().buildRecordsFromJson(content, null, null);
default -> throw (new QException("Unexpected file format: " + fileFormat));
};
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// now, wrap those records with the fields of the importRecord table, putting the unknown fields in a blob together //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> importRecordList = new ArrayList<>();
int recordNo = 1;
for(QRecord record : contentRecords)
{
record.setValue("recordNo", recordNo++);
// todo - client_id??
importRecordList.add(record);
}
return (importRecordList);
}
/*******************************************************************************
**
*******************************************************************************/
private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
{
List<F> files = actionBase.listFiles(table, backend);
Map<String, F> rs = new LinkedHashMap<>();
for(F file : files)
{
String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
rs.put(fileName, file);
}
return (rs);
}
}

82
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizer.java Normal file
View File

@ -0,0 +1,82 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** combine all unstructured fields of the record into a JSON blob in the "values" field.
*******************************************************************************/
public class ImportRecordPostQueryCustomizer extends AbstractPostQueryCustomizer
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> apply(List<QRecord> records)
{
if(CollectionUtils.nullSafeHasContents(records))
{
QTableMetaData table = null;
if(StringUtils.hasContent(records.get(0).getTableName()))
{
table = QContext.getQInstance().getTable(records.get(0).getTableName());
}
for(QRecord record : records)
{
Map<String, Serializable> values = record.getValues();
if(table != null)
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// remove known values from a clone of the values map - then only put the un-structured values in a JSON document in the values field //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
values = new HashMap<>(values);
for(String fieldName : table.getFields().keySet())
{
values.remove(fieldName);
}
}
String valuesJson = JsonUtils.toJson(values);
record.setValue("values", valuesJson);
}
}
return (records);
}
}

56
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@ -38,12 +39,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed.StreamedETLFilesystemBackendStep; import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed.StreamedETLFilesystemBackendStep;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer.FilesystemImporterMetaDataTemplate;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer.FilesystemImporterProcessMetaDataBuilder;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
@ -59,16 +63,19 @@ public class TestUtils
public static final String BACKEND_NAME_S3 = "s3"; public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
public static final String BACKEND_NAME_MOCK = "mock"; public static final String BACKEND_NAME_MOCK = "mock";
public static final String BACKEND_NAME_MEMORY = "memory";
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob";
public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive";
public static final String TABLE_NAME_PERSON_S3 = "person-s3"; public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter";
/////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////
// shouldn't be accessed directly, as we append a counter to it. // // shouldn't be accessed directly, as we append a counter to it. //
@ -136,15 +143,30 @@ public class TestUtils
qInstance.addTable(defineLocalFilesystemJSONPersonTable()); qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addTable(defineLocalFilesystemCSVPersonTable()); qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addTable(defineLocalFilesystemBlobTable()); qInstance.addTable(defineLocalFilesystemBlobTable());
qInstance.addTable(defineLocalFilesystemArchiveTable());
qInstance.addBackend(defineS3Backend()); qInstance.addBackend(defineS3Backend());
qInstance.addBackend(defineS3BackendSansPrefix()); qInstance.addBackend(defineS3BackendSansPrefix());
qInstance.addTable(defineS3CSVPersonTable()); qInstance.addTable(defineS3CSVPersonTable());
qInstance.addTable(defineS3BlobTable()); qInstance.addTable(defineS3BlobTable());
qInstance.addTable(defineS3BlobSansPrefixTable()); qInstance.addTable(defineS3BlobSansPrefixTable());
qInstance.addBackend(defineMockBackend()); qInstance.addBackend(defineMockBackend());
qInstance.addBackend(defineMemoryBackend());
qInstance.addTable(defineMockPersonTable()); qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
String importBaseName = "personImporter";
FilesystemImporterProcessMetaDataBuilder filesystemImporterProcessMetaDataBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
.withSourceTableName(TABLE_NAME_PERSON_LOCAL_FS_CSV)
.withFileFormat("csv")
.withArchiveFileEnabled(true)
.withArchiveTableName(TABLE_NAME_ARCHIVE_LOCAL_FS)
.withArchivePath("archive-of/personImporterFiles")
.withName(LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
FilesystemImporterMetaDataTemplate filesystemImporterMetaDataTemplate = new FilesystemImporterMetaDataTemplate(qInstance, importBaseName, BACKEND_NAME_MEMORY, filesystemImporterProcessMetaDataBuilder, table -> table.withAuditRules(QAuditRules.defaultInstanceLevelNone()));
filesystemImporterMetaDataTemplate.addToInstance(qInstance);
return (qInstance); return (qInstance);
} }
@ -257,6 +279,28 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemArchiveTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_ARCHIVE_LOCAL_FS)
.withLabel("Archive")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("archive")
.withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")
.withContentsFieldName("contents")
);
}
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
@ -356,6 +400,18 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QBackendMetaData defineMemoryBackend()
{
return (new QBackendMetaData()
.withBackendType(MemoryBackendModule.class)
.withName(BACKEND_NAME_MEMORY));
}
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/

114
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java Normal file
View File

@ -0,0 +1,114 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
import java.io.File;
import java.time.LocalDateTime;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import org.json.JSONObject;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for FilesystemImporterStep
*******************************************************************************/
class FilesystemImporterStepTest extends FilesystemActionTest
{
//////////////////////////////////////////////////////////////////////////
// note - we take advantage of the @BeforeEach and @AfterEach to set up //
// and clean up files on disk for this test. //
//////////////////////////////////////////////////////////////////////////
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void filesystemBaseAfterEach() throws Exception
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
new RunProcessAction().execute(runProcessInput);
String importBaseName = "personImporter";
assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
QRecord record = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
assertEquals(1, record.getValue("importFileId"));
assertEquals("John", record.getValue("firstName"));
assertThat(record.getValue("values")).isInstanceOf(String.class);
JSONObject values = new JSONObject(record.getValueString("values"));
assertEquals("John", values.get("firstName"));
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
String basePath = backend.getBasePath();
System.out.println(basePath);
///////////////////////////////////////////
// make sure 2 archive files got created //
///////////////////////////////////////////
LocalDateTime now = LocalDateTime.now();
File[] files = new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth()).listFiles();
assertNotNull(files);
assertEquals(2, files.length);
}
// todo - test json
// todo - test no files found
// todo - confirm delete happens?
// todo - updates?
}

85
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizerTest.java Normal file
View File

@ -0,0 +1,85 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for ImportRecordPostQueryCustomizer
*******************************************************************************/
class ImportRecordPostQueryCustomizerTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test()
{
Instant createDate = Instant.parse("2024-01-08T20:07:21Z");
List<QRecord> output = new ImportRecordPostQueryCustomizer().apply(List.of(
new QRecord()
.withTableName("personImporterImportRecord")
.withValue("importFileId", 1)
.withValue("unmapped", 2)
.withValue("unstructured", 3)
.withValue("nosqlObject", MapBuilder.of(HashMap::new).with("foo", "bar").with("createDate", createDate).build())
));
assertEquals(1, output.get(0).getValue("importFileId"));
assertEquals(2, output.get(0).getValue("unmapped"));
assertEquals(3, output.get(0).getValue("unstructured"));
assertEquals(Map.of("foo", "bar", "createDate", createDate), output.get(0).getValue("nosqlObject"));
///////////////////////////////////////////////////////////////////////////////////////////
// make sure all un-structured fields get put in the "values" field as a JSON string //
// compare as maps, beacuse JSONObject seems to care about the ordering, which, we don't //
///////////////////////////////////////////////////////////////////////////////////////////
Map<String, Object> expectedMap = new JSONObject("""
{
"unmapped": 2,
"unstructured": 3,
"nosqlObject":
{
"foo": "bar",
"createDate": "%s"
}
}
""".formatted(createDate)).toMap();
Map<String, Object> actualMap = new JSONObject(output.get(0).getValueString("values")).toMap();
assertThat(actualMap).isEqualTo(expectedMap);
}
}