From f0c07caba8fbbe422c4f236a8a4d4954efa7641e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 12 Feb 2025 14:21:02 -0600 Subject: [PATCH 01/67] Quality-of-life, add some todos for ideas --- .../model/actions/audits/AuditSingleInput.java | 14 ++++++++++++++ .../core/model/metadata/audits/AuditLevel.java | 1 + .../bulk/insert/mapping/BulkLoadValueMapper.java | 5 +++++ .../columnstats/ColumnStatsStep.java | 4 ++++ .../module/rdbms/actions/RDBMSInsertAction.java | 9 +++++++++ 5 files changed, 33 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java index 83b8785c..35b09c45 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java @@ -326,6 +326,20 @@ public class AuditSingleInput implements Serializable + /******************************************************************************* + ** Fluent setter for details + *******************************************************************************/ + public AuditSingleInput withDetailMessages(List details) + { + for(String detail : details) + { + addDetail(message); + } + return (this); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java index e80e2d68..185b0035 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java @@ -30,4 +30,5 @@ public enum AuditLevel NONE, RECORD, FIELD + // idea: only audit changes to fields, e.g., on edit. though, is that a different dimension than this? } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java index 626b8756..a0f90110 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -258,6 +258,11 @@ public class BulkLoadValueMapper valuesNotFound.add(value); } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - we should probably be doing a lot of what QJavalinImplementation.finishPossibleValuesRequest does here // + // to apply possible-value filters. difficult to pass values in, but needed... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + searchPossibleValueSourceInput.setIdList(idList); searchPossibleValueSourceInput.setLimit(values.size()); LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0))); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index cbe1e1a9..47c8c036 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -239,6 +239,10 @@ public class ColumnStatsStep implements BackendStep QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(); qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, queryJoin == null ? null : List.of(queryJoin), null); + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // todo - be aware of possible name collisions here!! (e.g., a table w/ a field named `count`) // + ///////////////////////////////////////////////////////////////////////////////////////////////// QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "count", countField), valueCounts); runBackendStepOutput.addValue("valueCounts", valueCounts); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index d6b89246..d9749d86 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -132,6 +133,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(QRecord record : page) { QRecord outputRecord = new QRecord(record); + if(!StringUtils.hasContent(outputRecord.getTableName())) + { + outputRecord.setTableName(tableName); + } outputRecords.add(outputRecord); } continue; @@ -151,6 +156,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(QRecord record : page) { QRecord outputRecord = new QRecord(record); + if(!StringUtils.hasContent(outputRecord.getTableName())) + { + outputRecord.setTableName(tableName); + } outputRecords.add(outputRecord); if(CollectionUtils.nullSafeIsEmpty(record.getErrors())) From be25fc1272d9d877f722d7e7c1ad75939cf0eb74 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 19:55:04 -0600 Subject: [PATCH 02/67] Refactor setup of backend variants to use a dedicated sub-object, with more flexible "backend setting" fields as a map based with enum keys, rather than dedicated set of methods --- .../actions/processes/RunProcessAction.java | 5 +- .../core/instances/QInstanceValidator.java | 52 +++- .../core/model/metadata/QBackendMetaData.java | 235 ++++++++---------- .../frontend/QFrontendTableMetaData.java | 3 +- .../variants/BackendVariantSetting.java | 31 +++ .../variants/BackendVariantsConfig.java | 189 ++++++++++++++ .../variants/LegacyBackendVariantSetting.java | 39 +++ .../core/scheduler/QScheduleManager.java | 11 +- .../core/scheduler/SchedulerUtils.java | 10 +- .../instances/QInstanceValidatorTest.java | 68 +++++ 10 files changed, 501 insertions(+), 142 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index a535ae47..806b0f5c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -815,13 +815,14 @@ public class RunProcessAction { QSession session = QContext.getQSession(); QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend()); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) { LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"); } else { - basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue()); + basepullKeyValue += "-" + session.getBackendVariants().get(variantTypeKey); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index dcc06391..fa4d1a23 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -108,12 +108,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.Automatio import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; +import org.apache.commons.lang.BooleanUtils; import org.quartz.CronExpression; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -543,6 +546,51 @@ public class QInstanceValidator { assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + "."); + /////////////////////// + // validate variants // + /////////////////////// + BackendVariantsConfig backendVariantsConfig = backend.getBackendVariantsConfig(); + if(BooleanUtils.isTrue(backend.getUsesVariants())) + { + if(assertCondition(backendVariantsConfig != null, "Missing backendVariantsConfig in backend [" + backendName + "] which is marked as usesVariants")) + { + assertCondition(StringUtils.hasContent(backendVariantsConfig.getVariantTypeKey()), "Missing variantTypeKey in backendVariantsConfig in [" + backendName + "]"); + + String optionsTableName = backendVariantsConfig.getOptionsTableName(); + QTableMetaData optionsTable = qInstance.getTable(optionsTableName); + if(assertCondition(StringUtils.hasContent(optionsTableName), "Missing optionsTableName in backendVariantsConfig in [" + backendName + "]")) + { + if(assertCondition(optionsTable != null, "Unrecognized optionsTableName [" + optionsTableName + "] in backendVariantsConfig in [" + backendName + "]")) + { + QQueryFilter optionsFilter = backendVariantsConfig.getOptionsFilter(); + if(optionsFilter != null) + { + validateQueryFilter(qInstance, "optionsFilter in backendVariantsConfig in backend [" + backendName + "]: ", optionsTable, optionsFilter, null); + } + } + } + + Map backendSettingSourceFieldNameMap = backendVariantsConfig.getBackendSettingSourceFieldNameMap(); + if(assertCondition(CollectionUtils.nullSafeHasContents(backendSettingSourceFieldNameMap), "Missing or empty backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]")) + { + if(optionsTable != null) + { + for(Map.Entry entry : backendSettingSourceFieldNameMap.entrySet()) + { + assertCondition(optionsTable.getFields().containsKey(entry.getValue()), "Unrecognized fieldName [" + entry.getValue() + "] in backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]"); + } + } + } + } + } + else + { + assertCondition(backendVariantsConfig == null, "Should not have a backendVariantsConfig in backend [" + backendName + "] which is not marked as usesVariants"); + } + + /////////////////////////////////////////// + // let the backend do its own validation // + /////////////////////////////////////////// backend.performValidation(this); runPlugins(QBackendMetaData.class, backend, qInstance); @@ -1616,12 +1664,12 @@ public class QInstanceValidator for(QFieldMetaData field : process.getInputFields()) { - validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName()); + validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName() + " "); } for(QFieldMetaData field : process.getOutputFields()) { - validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName()); + validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName() + " "); } if(process.getCancelStep() != null) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index 52f6c95f..3d79e591 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -26,8 +26,13 @@ import java.util.HashSet; import java.util.Set; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.serialization.QBackendMetaDataDeserializer; import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; +import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -45,21 +50,18 @@ public class QBackendMetaData implements TopLevelMetaDataInterface private Set enabledCapabilities = new HashSet<>(); private Set disabledCapabilities = new HashSet<>(); - private Boolean usesVariants = false; - private String variantOptionsTableIdField; - private String variantOptionsTableNameField; - private String variantOptionsTableTypeField; - private String variantOptionsTableTypeValue; - private String variantOptionsTableUsernameField; - private String variantOptionsTablePasswordField; - private String variantOptionsTableApiKeyField; - private String variantOptionsTableClientIdField; - private String variantOptionsTableClientSecretField; - private String variantOptionsTableName; + private Boolean usesVariants = false; + private BackendVariantsConfig backendVariantsConfig; // todo - at some point, we may want to apply this to secret properties on subclasses? // @JsonFilter("secretsFilter") + @Deprecated(since = "Replaced by filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") + private String variantOptionsTableTypeField; // a field on which to filter the variant-options table, to limit which records in it are available as variants + + @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") + private String variantOptionsTableTypeValue; // value for the type-field, to limit which records in it are available as variants; but also, the key in the session.backendVariants map! + /******************************************************************************* @@ -394,22 +396,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface - /******************************************************************************* - ** Getter for variantOptionsTableIdField - *******************************************************************************/ - public String getVariantOptionsTableIdField() - { - return (this.variantOptionsTableIdField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableIdField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key") public void setVariantOptionsTableIdField(String variantOptionsTableIdField) { - this.variantOptionsTableIdField = variantOptionsTableIdField; + ///////////////////////////////////////////////// + // noop as we migrate to backendVariantsConfig // + ///////////////////////////////////////////////// } @@ -417,30 +412,24 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableIdField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key") public QBackendMetaData withVariantOptionsTableIdField(String variantOptionsTableIdField) { - this.variantOptionsTableIdField = variantOptionsTableIdField; + this.setVariantOptionsTableIdField(variantOptionsTableIdField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableNameField - *******************************************************************************/ - public String getVariantOptionsTableNameField() - { - return (this.variantOptionsTableNameField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableNameField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel") public void setVariantOptionsTableNameField(String variantOptionsTableNameField) { - this.variantOptionsTableNameField = variantOptionsTableNameField; + ///////////////////////////////////////////////// + // noop as we migrate to backendVariantsConfig // + ///////////////////////////////////////////////// } @@ -448,30 +437,28 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableNameField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel") public QBackendMetaData withVariantOptionsTableNameField(String variantOptionsTableNameField) { - this.variantOptionsTableNameField = variantOptionsTableNameField; + this.setVariantOptionsTableNameField(variantOptionsTableNameField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableTypeField - *******************************************************************************/ - public String getVariantOptionsTableTypeField() - { - return (this.variantOptionsTableTypeField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableTypeField *******************************************************************************/ + @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") public void setVariantOptionsTableTypeField(String variantOptionsTableTypeField) { + this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeField); + this.variantOptionsTableTypeField = variantOptionsTableTypeField; + if(this.variantOptionsTableTypeValue != null) + { + this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue))); + } } @@ -479,30 +466,26 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableTypeField *******************************************************************************/ + @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") public QBackendMetaData withVariantOptionsTableTypeField(String variantOptionsTableTypeField) { - this.variantOptionsTableTypeField = variantOptionsTableTypeField; + this.setVariantOptionsTableTypeField(variantOptionsTableTypeField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableTypeValue - *******************************************************************************/ - public String getVariantOptionsTableTypeValue() - { - return (this.variantOptionsTableTypeValue); - } - - - /******************************************************************************* ** Setter for variantOptionsTableTypeValue *******************************************************************************/ + @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") public void setVariantOptionsTableTypeValue(String variantOptionsTableTypeValue) { this.variantOptionsTableTypeValue = variantOptionsTableTypeValue; + if(this.variantOptionsTableTypeField != null) + { + this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue))); + } } @@ -510,30 +493,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableTypeValue *******************************************************************************/ + @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") public QBackendMetaData withVariantOptionsTableTypeValue(String variantOptionsTableTypeValue) { - this.variantOptionsTableTypeValue = variantOptionsTableTypeValue; + this.setVariantOptionsTableTypeValue(variantOptionsTableTypeValue); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableUsernameField - *******************************************************************************/ - public String getVariantOptionsTableUsernameField() - { - return (this.variantOptionsTableUsernameField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableUsernameField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableUsernameField(String variantOptionsTableUsernameField) { - this.variantOptionsTableUsernameField = variantOptionsTableUsernameField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.USERNAME, variantOptionsTableUsernameField); } @@ -541,30 +516,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableUsernameField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableUsernameField(String variantOptionsTableUsernameField) { - this.variantOptionsTableUsernameField = variantOptionsTableUsernameField; + this.setVariantOptionsTableUsernameField(variantOptionsTableUsernameField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTablePasswordField - *******************************************************************************/ - public String getVariantOptionsTablePasswordField() - { - return (this.variantOptionsTablePasswordField); - } - - - /******************************************************************************* ** Setter for variantOptionsTablePasswordField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTablePasswordField(String variantOptionsTablePasswordField) { - this.variantOptionsTablePasswordField = variantOptionsTablePasswordField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.PASSWORD, variantOptionsTablePasswordField); } @@ -572,30 +539,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTablePasswordField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTablePasswordField(String variantOptionsTablePasswordField) { - this.variantOptionsTablePasswordField = variantOptionsTablePasswordField; + this.setVariantOptionsTablePasswordField(variantOptionsTablePasswordField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableApiKeyField - *******************************************************************************/ - public String getVariantOptionsTableApiKeyField() - { - return (this.variantOptionsTableApiKeyField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableApiKeyField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField) { - this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.API_KEY, variantOptionsTableApiKeyField); } @@ -603,30 +562,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableApiKeyField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField) { - this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField; + this.setVariantOptionsTableApiKeyField(variantOptionsTableApiKeyField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableName - *******************************************************************************/ - public String getVariantOptionsTableName() - { - return (this.variantOptionsTableName); - } - - - /******************************************************************************* ** Setter for variantOptionsTableName *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.tableName") public void setVariantOptionsTableName(String variantOptionsTableName) { - this.variantOptionsTableName = variantOptionsTableName; + this.getOrWithNewBackendVariantsConfig().withOptionsTableName(variantOptionsTableName); } @@ -634,9 +585,10 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableName *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.tableName") public QBackendMetaData withVariantOptionsTableName(String variantOptionsTableName) { - this.variantOptionsTableName = variantOptionsTableName; + this.setVariantOptionsTableName(variantOptionsTableName); return (this); } @@ -651,22 +603,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface qInstance.addBackend(this); } - /******************************************************************************* - ** Getter for variantOptionsTableClientIdField - *******************************************************************************/ - public String getVariantOptionsTableClientIdField() - { - return (this.variantOptionsTableClientIdField); - } - /******************************************************************************* ** Setter for variantOptionsTableClientIdField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField) { - this.variantOptionsTableClientIdField = variantOptionsTableClientIdField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_ID, variantOptionsTableClientIdField); } @@ -674,30 +619,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableClientIdField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField) { - this.variantOptionsTableClientIdField = variantOptionsTableClientIdField; + this.setVariantOptionsTableClientIdField(variantOptionsTableClientIdField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableClientSecretField - *******************************************************************************/ - public String getVariantOptionsTableClientSecretField() - { - return (this.variantOptionsTableClientSecretField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableClientSecretField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField) { - this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_SECRET, variantOptionsTableClientSecretField); } @@ -705,11 +642,55 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableClientSecretField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField) { - this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField; + this.setVariantOptionsTableClientSecretField(variantOptionsTableClientSecretField); return (this); } + + /******************************************************************************* + ** Getter for backendVariantsConfig + *******************************************************************************/ + public BackendVariantsConfig getBackendVariantsConfig() + { + return (this.backendVariantsConfig); + } + + + + /******************************************************************************* + ** Setter for backendVariantsConfig + *******************************************************************************/ + public void setBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig) + { + this.backendVariantsConfig = backendVariantsConfig; + } + + + + /******************************************************************************* + ** Fluent setter for backendVariantsConfig + *******************************************************************************/ + public QBackendMetaData withBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig) + { + this.backendVariantsConfig = backendVariantsConfig; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private BackendVariantsConfig getOrWithNewBackendVariantsConfig() + { + if(backendVariantsConfig == null) + { + setBackendVariantsConfig(new BackendVariantsConfig()); + } + return backendVariantsConfig; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java index 6e98fdec..01b7832e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -86,7 +86,6 @@ public class QFrontendTableMetaData ////////////////////////////////////////////////////////////////////////////////// - /******************************************************************************* ** *******************************************************************************/ @@ -170,7 +169,7 @@ public class QFrontendTableMetaData if(backend != null && backend.getUsesVariants()) { usesVariants = true; - variantTableLabel = QContext.getQInstance().getTable(backend.getVariantOptionsTableName()).getLabel(); + variantTableLabel = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName()).getLabel(); } this.helpContents = tableMetaData.getHelpContent(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java new file mode 100644 index 00000000..ea57a3ed --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java @@ -0,0 +1,31 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +/******************************************************************************* + ** interface to be implemented by enums (presumably) that define the possible + ** settings a particular backend type can get from a variant record. + *******************************************************************************/ +public interface BackendVariantSetting +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java new file mode 100644 index 00000000..20237e7a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java @@ -0,0 +1,189 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.util.HashMap; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Configs for how a backend that uses variants works. Specifically: + ** + ** - the variant "type key" - e.g., key for variants map in session. + ** - what table supplies the variant options (optionsTableName + ** - an optional filter to apply to that options table + ** - a map of the settings that a backend gets from its variant table to the + ** field names in that table that they come from. e.g., a backend may have a + ** username attribute, whose value comes from a field named "theUser" in the + ** variant options table. + *******************************************************************************/ +public class BackendVariantsConfig +{ + private String variantTypeKey; + + private String optionsTableName; + private QQueryFilter optionsFilter; + + private Map backendSettingSourceFieldNameMap; + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getOptionsTableName() + { + return (this.optionsTableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setOptionsTableName(String optionsTableName) + { + this.optionsTableName = optionsTableName; + } + + + + /******************************************************************************* + ** Getter for filter + *******************************************************************************/ + public QQueryFilter getOptionsFilter() + { + return (this.optionsFilter); + } + + + + /******************************************************************************* + ** Setter for filter + *******************************************************************************/ + public void setOptionsFilter(QQueryFilter optionsFilter) + { + this.optionsFilter = optionsFilter; + } + + + + /******************************************************************************* + ** Getter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public Map getBackendSettingSourceFieldNameMap() + { + return (this.backendSettingSourceFieldNameMap); + } + + + + /******************************************************************************* + ** Setter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public void setBackendSettingSourceFieldNameMap(Map backendSettingSourceFieldNameMap) + { + this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap; + } + + + + /******************************************************************************* + ** Fluent setter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public BackendVariantsConfig withBackendSettingSourceFieldName(BackendVariantSetting backendVariantSetting, String sourceFieldName) + { + if(this.backendSettingSourceFieldNameMap == null) + { + this.backendSettingSourceFieldNameMap = new HashMap<>(); + } + this.backendSettingSourceFieldNameMap.put(backendVariantSetting, sourceFieldName); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public BackendVariantsConfig withBackendSettingSourceFieldNameMap(Map backendSettingSourceFieldNameMap) + { + this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for variantTypeKey + *******************************************************************************/ + public String getVariantTypeKey() + { + return (this.variantTypeKey); + } + + + + /******************************************************************************* + ** Setter for variantTypeKey + *******************************************************************************/ + public void setVariantTypeKey(String variantTypeKey) + { + this.variantTypeKey = variantTypeKey; + } + + + + /******************************************************************************* + ** Fluent setter for variantTypeKey + *******************************************************************************/ + public BackendVariantsConfig withVariantTypeKey(String variantTypeKey) + { + this.variantTypeKey = variantTypeKey; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for optionsTableName + *******************************************************************************/ + public BackendVariantsConfig withOptionsTableName(String optionsTableName) + { + this.optionsTableName = optionsTableName; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for optionsFilter + *******************************************************************************/ + public BackendVariantsConfig withOptionsFilter(QQueryFilter optionsFilter) + { + this.optionsFilter = optionsFilter; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java new file mode 100644 index 00000000..59ccc38c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +/******************************************************************************* + ** temporary class, while we migrate from original way that variants were set up + ** e.g., by calling 'variantOptionsTableUsernameField', to the new way, using + ** the BackendVariantsConfig which uses a map of enum constants. + ** + ** so when those deprecated setters are removed, this enum can be too. + *****************************************************************************/ +public enum LegacyBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + API_KEY, + CLIENT_ID, + CLIENT_SECRET +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index 3dfec3ae..88246600 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -441,11 +441,16 @@ public class QScheduleManager try { HashMap parameters = new HashMap<>(paramMap); - HashMap variantMap = new HashMap<>(Map.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + String variantOptionsTableName = backendMetaData.getBackendVariantsConfig().getOptionsTableName(); + String variantOptionsTableIdFieldName = QContext.getQInstance().getTable(variantOptionsTableName).getPrimaryKeyField(); + + HashMap variantMap = new HashMap<>(Map.of(variantTypeKey, qRecord.getValue(variantOptionsTableIdFieldName))); parameters.put("backendVariantData", variantMap); - String identity = schedulableIdentity.getIdentity() + ";" + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); - String description = schedulableIdentity.getDescription() + " for variant: " + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); + String identity = schedulableIdentity.getIdentity() + ";" + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName); + String description = schedulableIdentity.getDescription() + " for variant: " + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName); BasicSchedulableIdentity variantIdentity = new BasicSchedulableIdentity(identity, description); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java index c7cc6a4c..e7b46ea3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java @@ -34,9 +34,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -102,7 +99,8 @@ public class SchedulerUtils try { QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend()); - Map thisVariantData = MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())); + QTableMetaData variantTable = QContext.getQInstance().getTable(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + Map thisVariantData = MapBuilder.of(backendMetaData.getBackendVariantsConfig().getVariantTypeKey(), qRecord.getValue(variantTable.getPrimaryKeyField())); executeSingleProcess(process, thisVariantData, processInputValues); } catch(Exception e) @@ -181,8 +179,8 @@ public class SchedulerUtils QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(processMetaData.getVariantBackend()); QueryInput queryInput = new QueryInput(); - queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue()))); + queryInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + queryInput.setFilter(backendMetaData.getBackendVariantsConfig().getOptionsFilter()); QueryOutput queryOutput = new QueryAction().execute(queryInput); records = queryOutput.getRecords(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index c09d1ac8..9ce5d9f6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -55,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; @@ -93,6 +95,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; @@ -182,6 +186,70 @@ public class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBackendVariants() + { + BackendVariantSetting setting = new BackendVariantSetting() {}; + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true)), + "Missing backendVariantsConfig in backend [variant] which is marked as usesVariants"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(false) + .withBackendVariantsConfig(new BackendVariantsConfig())), + "Should not have a backendVariantsConfig"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(null) + .withBackendVariantsConfig(new BackendVariantsConfig())), + "Should not have a backendVariantsConfig"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig())), + "Missing variantTypeKey in backendVariantsConfig", + "Missing optionsTableName in backendVariantsConfig", + "Missing or empty backendSettingSourceFieldNameMap"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName("notATable") + .withBackendSettingSourceFieldNameMap(Map.of(setting, "field")))), + "Unrecognized optionsTableName [notATable] in backendVariantsConfig"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withOptionsFilter(new QQueryFilter(new QFilterCriteria("notAField", QCriteriaOperator.EQUALS, 1))) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")))), + "optionsFilter in backendVariantsConfig in backend [variant]: Criteria fieldName notAField is not a field"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "noSuchField")))), + "Unrecognized fieldName [noSuchField] in backendSettingSourceFieldNameMap"); + } + + + /******************************************************************************* ** Test an instance with null tables - should throw. ** From 4c502df3288059678d26b888a274345fbc9e4a58 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:01:00 -0600 Subject: [PATCH 03/67] Update to use new backendVariantConfig; removed unused session field in base api action --- .../module/api/actions/AbstractAPIAction.java | 1 - .../module/api/actions/BaseAPIActionUtil.java | 38 ++++++++++++++----- .../metadata/APIBackendVariantSetting.java | 38 +++++++++++++++++++ .../javalin/QJavalinImplementation.java | 15 ++++---- 4 files changed, 75 insertions(+), 17 deletions(-) create mode 100644 qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java index 8425fbfb..6a32ae17 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java @@ -61,7 +61,6 @@ public abstract class AbstractAPIAction apiActionUtil.setBackendMetaData(this.backendMetaData); apiActionUtil.setActionInput(actionInput); - apiActionUtil.setSession(session); } } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 209285e0..3623db11 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -61,6 +61,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -77,6 +79,7 @@ import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorExcep import com.kingsrook.qqq.backend.module.api.model.AuthorizationType; import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog; import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData; +import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendVariantSetting; import com.kingsrook.qqq.backend.module.api.model.metadata.APITableBackendDetails; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.BooleanUtils; @@ -114,7 +117,6 @@ public class BaseAPIActionUtil { private final QLogger LOG = QLogger.getLogger(BaseAPIActionUtil.class); - protected QSession session; // todo not commit - delete!! protected APIBackendMetaData backendMetaData; protected AbstractTableActionInput actionInput; @@ -778,7 +780,7 @@ public class BaseAPIActionUtil if(backendMetaData.getUsesVariants()) { QRecord record = getVariantRecord(); - return (record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField())); + return (record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.API_KEY, APIBackendVariantSetting.API_KEY))); } return (backendMetaData.getApiKey()); @@ -786,6 +788,18 @@ public class BaseAPIActionUtil + /*************************************************************************** + ** todo - once deprecated variant methods are removed from QBackendMetaData, + ** then we can remove the LegacyBackendVariantSetting enum, and this param. + ***************************************************************************/ + private String getVariantSettingSourceFieldName(APIBackendMetaData backendMetaData, LegacyBackendVariantSetting legacyBackendVariantSetting, APIBackendVariantSetting apiBackendVariantSetting) + { + Map map = CollectionUtils.nonNullMap(backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap()); + return map.getOrDefault(legacyBackendVariantSetting, map.get(apiBackendVariantSetting)); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -794,7 +808,10 @@ public class BaseAPIActionUtil if(backendMetaData.getUsesVariants()) { QRecord record = getVariantRecord(); - return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField()))); + return (Pair.of( + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.USERNAME, APIBackendVariantSetting.USERNAME)), + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.PASSWORD, APIBackendVariantSetting.PASSWORD)) + )); } return (Pair.of(backendMetaData.getUsername(), backendMetaData.getPassword())); @@ -812,14 +829,14 @@ public class BaseAPIActionUtil Serializable variantId = getVariantId(); GetInput getInput = new GetInput(); getInput.setShouldMaskPasswords(false); - getInput.setTableName(backendMetaData.getVariantOptionsTableName()); + getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); getInput.setPrimaryKey(variantId); GetOutput getOutput = new GetAction().execute(getInput); QRecord record = getOutput.getRecord(); if(record == null) { - throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'")); + throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); } return record; } @@ -832,11 +849,11 @@ public class BaseAPIActionUtil protected Serializable getVariantId() throws QException { QSession session = QContext.getQSession(); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getBackendVariantsConfig().getVariantTypeKey())) { throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); } - Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue()); + Serializable variantId = session.getBackendVariants().get(backendMetaData.getBackendVariantsConfig().getVariantTypeKey()); return variantId; } @@ -945,7 +962,10 @@ public class BaseAPIActionUtil if(backendMetaData.getUsesVariants()) { QRecord record = getVariantRecord(); - return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableClientIdField()), record.getValueString(backendMetaData.getVariantOptionsTableClientSecretField()))); + return (Pair.of( + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_ID, APIBackendVariantSetting.CLIENT_ID)), + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_SECRET, APIBackendVariantSetting.CLIENT_SECRET)) + )); } return (Pair.of(backendMetaData.getClientId(), backendMetaData.getClientSecret())); @@ -1480,9 +1500,9 @@ public class BaseAPIActionUtil ** Setter for session ** *******************************************************************************/ + @Deprecated(since = "wasn't used.") public void setSession(QSession session) { - this.session = session; } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java new file mode 100644 index 00000000..681977d3 --- /dev/null +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.api.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; + + +/******************************************************************************* + ** settings that the API backend module can get from a backend variant. + *******************************************************************************/ +public enum APIBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + API_KEY, + CLIENT_ID, + CLIENT_SECRET +} diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index c82274d6..80f227d3 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -95,8 +95,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; @@ -1200,15 +1198,18 @@ public class QJavalinImplementation ///////////////////////////////////////////////////////////////////////////////////// if(backend != null && backend.getUsesVariants()) { - queryInput.setTableName(backend.getVariantOptionsTableName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backend.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backend.getVariantOptionsTableTypeValue()))); + QTableMetaData variantsTable = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName()); + + queryInput.setTableName(variantsTable.getName()); + queryInput.setFilter(backend.getBackendVariantsConfig().getOptionsFilter()); + queryInput.setShouldGenerateDisplayValues(true); QueryOutput output = new QueryAction().execute(queryInput); for(QRecord qRecord : output.getRecords()) { variants.add(new QFrontendVariant() - .withId(qRecord.getValue(backend.getVariantOptionsTableIdField())) - .withType(backend.getVariantOptionsTableTypeValue()) - .withName(qRecord.getValueString(backend.getVariantOptionsTableNameField()))); + .withId(qRecord.getValue(variantsTable.getPrimaryKeyField())) + .withType(backend.getBackendVariantsConfig().getVariantTypeKey()) + .withName(qRecord.getRecordLabel())); } QJavalinAccessLogger.logStartSilent("variants"); From bacfa57c5e6aa1e7bcaa3154b1ef18dfc886ebc3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:07:37 -0600 Subject: [PATCH 04/67] New ways of working with field sections --- .../model/metadata/tables/QTableMetaData.java | 20 ++ .../model/metadata/tables/SectionFactory.java | 221 ++++++++++++++++++ .../metadata/tables/SectionFactoryTest.java | 62 +++++ 3 files changed, 303 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 93ae4f96..1df589d1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -712,6 +713,25 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData + /******************************************************************************* + ** Getter for sections + ** + *******************************************************************************/ + public QFieldSection getSection(String name) + { + for(QFieldSection qFieldSection : CollectionUtils.nonNullList(sections)) + { + if(qFieldSection.getName().equals(name)) + { + return (qFieldSection); + } + } + + return (null); + } + + + /******************************************************************************* ** Setter for sections ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java new file mode 100644 index 00000000..52f84d78 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java @@ -0,0 +1,221 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; + + +/******************************************************************************* + ** Factory class for creating "standard" qfield sections. e.g., if you want + ** the same t1, t2, and t3 section on all your tables, use this class to + ** produce them. + ** + ** You can change the default name & iconNames for those sections, but note, + ** this is a static/utility style class, so those settings are static fields. + ** + ** The method customT2 is provided as not much of a shortcut over "doing it yourself", + ** but to allow all sections for a table to be produced through calls to this factory, + ** so they look more similar. + *******************************************************************************/ +public class SectionFactory +{ + private static String defaultT1name = "identity"; + private static String defaultT1iconName = "badge"; + private static String defaultT2name = "data"; + private static String defaultT2iconName = "text_snippet"; + private static String defaultT3name = "dates"; + private static String defaultT3iconName = "calendar_month"; + + + /******************************************************************************* + ** private constructor, to enforce static usage, e.g., to make clear the fields + ** are static fields. + ** + *******************************************************************************/ + private SectionFactory() + { + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection defaultT1(String... fieldNames) + { + return new QFieldSection(defaultT1name, new QIcon().withName(defaultT1iconName), Tier.T1, List.of(fieldNames)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection defaultT2(String... fieldNames) + { + return new QFieldSection(defaultT2name, new QIcon().withName(defaultT2iconName), Tier.T2, List.of(fieldNames)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection customT2(String name, QIcon icon, String... fieldNames) + { + return new QFieldSection(name, icon, Tier.T2, List.of(fieldNames)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection defaultT3(String... fieldNames) + { + return new QFieldSection(defaultT3name, new QIcon().withName(defaultT3iconName), Tier.T3, List.of(fieldNames)); + } + + + + /******************************************************************************* + ** Getter for defaultT1name + *******************************************************************************/ + public static String getDefaultT1name() + { + return (SectionFactory.defaultT1name); + } + + + + /******************************************************************************* + ** Setter for defaultT1name + *******************************************************************************/ + public static void setDefaultT1name(String defaultT1name) + { + SectionFactory.defaultT1name = defaultT1name; + } + + + + /******************************************************************************* + ** Getter for defaultT1iconName + *******************************************************************************/ + public static String getDefaultT1iconName() + { + return (SectionFactory.defaultT1iconName); + } + + + + /******************************************************************************* + ** Setter for defaultT1iconName + *******************************************************************************/ + public static void setDefaultT1iconName(String defaultT1iconName) + { + SectionFactory.defaultT1iconName = defaultT1iconName; + } + + + + /******************************************************************************* + ** Getter for defaultT2name + *******************************************************************************/ + public static String getDefaultT2name() + { + return (SectionFactory.defaultT2name); + } + + + + /******************************************************************************* + ** Setter for defaultT2name + *******************************************************************************/ + public static void setDefaultT2name(String defaultT2name) + { + SectionFactory.defaultT2name = defaultT2name; + } + + + + /******************************************************************************* + ** Getter for defaultT2iconName + *******************************************************************************/ + public static String getDefaultT2iconName() + { + return (SectionFactory.defaultT2iconName); + } + + + + /******************************************************************************* + ** Setter for defaultT2iconName + *******************************************************************************/ + public static void setDefaultT2iconName(String defaultT2iconName) + { + SectionFactory.defaultT2iconName = defaultT2iconName; + } + + + + /******************************************************************************* + ** Getter for defaultT3name + *******************************************************************************/ + public static String getDefaultT3name() + { + return (SectionFactory.defaultT3name); + } + + + + /******************************************************************************* + ** Setter for defaultT3name + *******************************************************************************/ + public static void setDefaultT3name(String defaultT3name) + { + SectionFactory.defaultT3name = defaultT3name; + } + + + + /******************************************************************************* + ** Getter for defaultT3iconName + *******************************************************************************/ + public static String getDefaultT3iconName() + { + return (SectionFactory.defaultT3iconName); + } + + + + /******************************************************************************* + ** Setter for defaultT3iconName + *******************************************************************************/ + public static void setDefaultT3iconName(String defaultT3iconName) + { + SectionFactory.defaultT3iconName = defaultT3iconName; + } + + + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java new file mode 100644 index 00000000..5f406acb --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for SectionFactory + *******************************************************************************/ +class SectionFactoryTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + QFieldSection t1section = SectionFactory.defaultT1("id", "name"); + assertEquals(SectionFactory.getDefaultT1name(), t1section.getName()); + assertEquals(SectionFactory.getDefaultT1iconName(), t1section.getIcon().getName()); + assertEquals(Tier.T1, t1section.getTier()); + assertEquals(List.of("id", "name"), t1section.getFieldNames()); + + QFieldSection t2section = SectionFactory.defaultT2("size", "age"); + assertEquals(SectionFactory.getDefaultT2name(), t2section.getName()); + assertEquals(SectionFactory.getDefaultT2iconName(), t2section.getIcon().getName()); + assertEquals(Tier.T2, t2section.getTier()); + assertEquals(List.of("size", "age"), t2section.getFieldNames()); + + QFieldSection t3section = SectionFactory.defaultT3("createDate", "modifyDate"); + assertEquals(SectionFactory.getDefaultT3name(), t3section.getName()); + assertEquals(SectionFactory.getDefaultT3iconName(), t3section.getIcon().getName()); + assertEquals(Tier.T3, t3section.getTier()); + assertEquals(List.of("createDate", "modifyDate"), t3section.getFieldNames()); + } + +} \ No newline at end of file From 7bd560b7a8cac5127431f6b9a5f20a3460e490b3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:09:13 -0600 Subject: [PATCH 05/67] Initial checkin --- .../metadata/EmptyMetaDataProducerOutput.java | 47 +++++++++++++++++++ .../EmptyMetaDataProducerOutputTest.java | 44 +++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java new file mode 100644 index 00000000..04c4e4eb --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java @@ -0,0 +1,47 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import com.kingsrook.qqq.backend.core.logging.QLogger; + + +/******************************************************************************* + ** for use-cases where a metaDataProducer directly adds its objects to the + ** qInstance, then this empty object can be returned. + *******************************************************************************/ +public class EmptyMetaDataProducerOutput implements MetaDataProducerOutput +{ + private static final QLogger LOG = QLogger.getLogger(EmptyMetaDataProducerOutput.class); + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void addSelfToInstance(QInstance instance) + { + ///////////////////////////////// + // noop - this output is empty // + ///////////////////////////////// + LOG.trace("empty meta data producer has nothing to add."); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java new file mode 100644 index 00000000..118c27d3 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for EmptyMetaDataProducerOutput + *******************************************************************************/ +class EmptyMetaDataProducerOutputTest +{ + + /******************************************************************************* + ** sorry, just here to avoid a dip in coverage. + *******************************************************************************/ + @Test + void test() + { + QInstance qInstance = new QInstance(); + new EmptyMetaDataProducerOutput().addSelfToInstance(qInstance); + } + +} \ No newline at end of file From 243cf66dbd4a27f086f7c1051a9f37e649a628ce Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:09:46 -0600 Subject: [PATCH 06/67] Avoid NPE on empty list of fields in setBlobValuesToDownloadUrls --- .../qqq/backend/core/actions/values/QValueFormatter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 3c695cb7..39a6a520 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -28,6 +28,7 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -508,7 +509,7 @@ public class QValueFormatter { @SuppressWarnings("unchecked") // instance validation should make this safe! List fileNameFormatFields = (List) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS); - List values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList(); + List values = CollectionUtils.nullSafeHasContents(fileNameFormatFields) ? fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList() : Collections.emptyList(); fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values); } } From 72e175e1a6de64303f562c5b25ee934d71109af9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:20:56 -0600 Subject: [PATCH 07/67] Add method to work with recordEntities --- .../processes/RunBackendStepInput.java | 19 +++++++++++++++++++ .../processes/RunBackendStepOutput.java | 13 ++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 17b3e608..764f930c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes; import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -34,9 +35,11 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface; @@ -247,6 +250,22 @@ public class RunBackendStepInput extends AbstractActionInput + /******************************************************************************* + ** Getter for records converted to entities of a given type. + ** + *******************************************************************************/ + public List getRecordsAsEntities(Class entityClass) throws QException + { + List rs = new ArrayList<>(); + for(QRecord record : processState.getRecords()) + { + rs.add(QRecordEntity.fromQRecord(entityClass, record)); + } + return (rs); + } + + + /******************************************************************************* ** Setter for records ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java index 7c89b98a..ba6b87c5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java @@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -258,7 +259,7 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial /******************************************************************************* - ** + ** add a record to the step output, e.g., for going through to the next step. *******************************************************************************/ public void addRecord(QRecord record) { @@ -271,6 +272,16 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial + /*************************************************************************** + ** add a RecordEntity to the step output, e.g., for going through to the next step. + ***************************************************************************/ + public void addRecordEntity(QRecordEntity recordEntity) + { + addRecord(recordEntity.toQRecord()); + } + + + /******************************************************************************* ** Getter for auditInputList *******************************************************************************/ From 2591e6ad44dbb58abf62da15f7bc594d19480e34 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:21:32 -0600 Subject: [PATCH 08/67] Update javadoc because i can't ever remember if inputStream or outputStream is used for writing or reading --- .../qqq/backend/core/actions/tables/StorageAction.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java index 5cbcfc7e..ad868105 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java @@ -47,7 +47,8 @@ public class StorageAction { /******************************************************************************* - ** + ** create an output stream in the storage backend - that can be written to, + ** for the purpose of inserting or writing a file into storage. *******************************************************************************/ public OutputStream createOutputStream(StorageInput storageInput) throws QException { @@ -59,7 +60,8 @@ public class StorageAction /******************************************************************************* - ** + ** create an input stream in the storage backend - that can be read from, + ** for the purpose of getting or reading a file from storage. *******************************************************************************/ public InputStream getInputStream(StorageInput storageInput) throws QException { From 5a7199495d6abb833d4fe62133dded69719291d8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:24:10 -0600 Subject: [PATCH 09/67] Basic support for variants; more fields on ONE type file records (size, dates); apply skip, limit, filter, sort on listings/queries for ONE-type files; treat contents as heavy-field if so set; more try-catch (e.g., upon write file) --- .../actions/AbstractBaseFilesystemAction.java | 376 +++++++++++++----- ...AbstractFilesystemTableBackendDetails.java | 96 +++++ .../FilesystemTableMetaDataBuilder.java | 26 +- .../actions/AbstractFilesystemAction.java | 46 +++ .../importer/FilesystemImporterStep.java | 3 +- .../module/filesystem/s3/S3BackendModule.java | 13 + .../s3/actions/AbstractS3Action.java | 36 +- 7 files changed, 489 insertions(+), 107 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index fdf343c6..a896230c 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -25,17 +25,25 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -47,12 +55,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -68,6 +81,8 @@ public abstract class AbstractBaseFilesystemAction { private static final QLogger LOG = QLogger.getLogger(AbstractBaseFilesystemAction.class); + protected QRecord backendVariantRecord = null; + /******************************************************************************* @@ -80,6 +95,21 @@ public abstract class AbstractBaseFilesystemAction + /*************************************************************************** + ** get the size of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Long getFileSize(FILE file); + + /*************************************************************************** + ** get the createDate of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Instant getFileCreateDate(FILE file); + + /*************************************************************************** + ** get the createDate of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Instant getFileModifyDate(FILE file); + /******************************************************************************* ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses. *******************************************************************************/ @@ -116,13 +146,20 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; + + /******************************************************************************* ** e.g., with a base path of /foo/ ** and a table path of /bar/ ** and a file at /foo/bar/baz.txt ** give us just the baz.txt part. *******************************************************************************/ - public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable); + public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); + return (strippedPath); + } @@ -133,7 +170,17 @@ public abstract class AbstractBaseFilesystemAction public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase) { AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase); - String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : ""; + + String basePath = metaData.getBasePath(); + if(backendBase.getUsesVariants()) + { + Map fieldNameMap = backendBase.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap(); + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.BASE_PATH)) + { + basePath = backendVariantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.BASE_PATH)); + } + } + String fullPath = StringUtils.hasContent(basePath) ? basePath : ""; AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); if(StringUtils.hasContent(tableDetails.getBasePath())) @@ -208,100 +255,14 @@ public abstract class AbstractBaseFilesystemAction AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); List files = listFiles(table, queryInput.getBackend(), queryInput.getFilter()); - int recordCount = 0; - - FILE_LOOP: - for(FILE file : files) + switch(tableDetails.getCardinality()) { - InputStream inputStream = readFile(file); - switch(tableDetails.getCardinality()) - { - case MANY: - { - LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file))); - switch(tableDetails.getRecordFormat()) - { - case CSV: - { - String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - fileContents = customizeFileContentsAfterReading(table, fileContents); - - if(queryInput.getRecordPipe() != null) - { - new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> - { - //////////////////////////////////////////////////////////////////////////////////////////// - // Before the records go into the pipe, make sure their backend details are added to them // - //////////////////////////////////////////////////////////////////////////////////////////// - addBackendDetailsToRecord(record, file); - })); - } - else - { - List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - queryOutput.addRecords(recordsInFile); - } - break; - } - case JSON: - { - String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - fileContents = customizeFileContentsAfterReading(table, fileContents); - - // todo - pipe support!! - List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - - queryOutput.addRecords(recordsInFile); - break; - } - default: - { - throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); - } - } - break; - } - case ONE: - { - //////////////////////////////////////////////////////////////////////////////// - // for one-record tables, put the entire file's contents into a single record // - //////////////////////////////////////////////////////////////////////////////// - String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); - byte[] bytes = inputStream.readAllBytes(); - - QRecord record = new QRecord() - .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase) - .withValue(tableDetails.getContentsFieldName(), bytes); - queryOutput.addRecord(record); - - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - // keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) // - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - recordCount++; - - //////////////////////////////////////////////////////////////////////////// - // break out of the file loop if we have hit the limit (if one was given) // - //////////////////////////////////////////////////////////////////////////// - if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null) - { - if(recordCount >= queryInput.getFilter().getLimit()) - { - break FILE_LOOP; - } - } - - break; - } - default: - { - throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); - } - } + case MANY -> completeExecuteQueryForManyTable(queryInput, queryOutput, files, table, tableDetails); + case ONE -> completeExecuteQueryForOneTable(queryInput, queryOutput, files, table, tableDetails); + default -> throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); } - return queryOutput; + return (queryOutput); } catch(Exception e) { @@ -312,6 +273,157 @@ public abstract class AbstractBaseFilesystemAction + /*************************************************************************** + ** + ***************************************************************************/ + private void completeExecuteQueryForOneTable(QueryInput queryInput, QueryOutput queryOutput, List files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException + { + int recordCount = 0; + List records = new ArrayList<>(); + + for(FILE file : files) + { + //////////////////////////////////////////////////////////////////////////////// + // for one-record tables, put the entire file's contents into a single record // + //////////////////////////////////////////////////////////////////////////////// + String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); + QRecord record = new QRecord().withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase); + + if(StringUtils.hasContent(tableDetails.getSizeFieldName())) + { + record.setValue(tableDetails.getSizeFieldName(), getFileSize(file)); + } + + if(StringUtils.hasContent(tableDetails.getCreateDateFieldName())) + { + record.setValue(tableDetails.getCreateDateFieldName(), getFileCreateDate(file)); + } + + if(StringUtils.hasContent(tableDetails.getModifyDateFieldName())) + { + record.setValue(tableDetails.getModifyDateFieldName(), getFileModifyDate(file)); + } + + if(shouldFileContentsBeRead(queryInput, table, tableDetails)) + { + try(InputStream inputStream = readFile(file)) + { + byte[] bytes = inputStream.readAllBytes(); + record.withValue(tableDetails.getContentsFieldName(), bytes); + } + catch(Exception e) + { + record.addError(new SystemErrorStatusMessage("Error reading file contents: " + e.getMessage())); + } + } + else + { + if(StringUtils.hasContent(tableDetails.getSizeFieldName())) + { + Long size = record.getValueLong(tableDetails.getSizeFieldName()); + if(size != null) + { + if(record.getBackendDetails() == null) + { + record.setBackendDetails(new HashMap<>()); + } + + if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null) + { + record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>()); + } + + ((Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size); + } + } + } + + if(BackendQueryFilterUtils.doesRecordMatch(queryInput.getFilter(), null, record)) + { + records.add(record); + } + } + + BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), records); + records = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), records); + queryOutput.addRecords(records); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void completeExecuteQueryForManyTable(QueryInput queryInput, QueryOutput queryOutput, List files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException, IOException + { + int recordCount = 0; + + for(FILE file : files) + { + try(InputStream inputStream = readFile(file)) + { + LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file))); + switch(tableDetails.getRecordFormat()) + { + case CSV -> + { + String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + if(queryInput.getRecordPipe() != null) + { + new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> + { + //////////////////////////////////////////////////////////////////////////////////////////// + // Before the records go into the pipe, make sure their backend details are added to them // + //////////////////////////////////////////////////////////////////////////////////////////// + addBackendDetailsToRecord(record, file); + })); + } + else + { + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + queryOutput.addRecords(recordsInFile); + } + } + case JSON -> + { + String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + // todo - pipe support!! + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + + queryOutput.addRecords(recordsInFile); + } + default -> throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static boolean shouldFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) + { + boolean doReadContents = true; + if(table.getField(tableDetails.getContentsFieldName()).getIsHeavy()) + { + if(!queryInput.getShouldFetchHeavyFields()) + { + doReadContents = false; + } + } + return doReadContents; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -319,7 +431,16 @@ public abstract class AbstractBaseFilesystemAction { QueryInput queryInput = new QueryInput(); queryInput.setTableName(countInput.getTableName()); - queryInput.setFilter(countInput.getFilter()); + + QQueryFilter filter = countInput.getFilter(); + if(filter != null) + { + filter = filter.clone(); + filter.setSkip(null); + filter.setLimit(null); + } + + queryInput.setFilter(filter); QueryOutput queryOutput = executeQuery(queryInput); CountOutput countOutput = new CountOutput(); @@ -353,11 +474,12 @@ public abstract class AbstractBaseFilesystemAction ** Method that subclasses can override to add pre-action things (e.g., setting up ** s3 client). *******************************************************************************/ - public void preAction(QBackendMetaData backendMetaData) + public void preAction(QBackendMetaData backendMetaData) throws QException { - ///////////////////////////////////////////////////////////////////// - // noop in base class - subclasses can add functionality if needed // - ///////////////////////////////////////////////////////////////////// + if(backendMetaData.getUsesVariants()) + { + this.backendVariantRecord = getVariantRecord(backendMetaData); + } } @@ -411,10 +533,18 @@ public abstract class AbstractBaseFilesystemAction { for(QRecord record : insertInput.getRecords()) { - String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); - writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); - record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); - output.addRecord(record); + try + { + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); + writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); + record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); + output.addRecord(record); + } + catch(Exception e) + { + record.addError(new SystemErrorStatusMessage("Error writing file: " + e.getMessage())); + output.addRecord(record); + } } } else @@ -429,4 +559,46 @@ public abstract class AbstractBaseFilesystemAction throw new QException("Error executing insert: " + e.getMessage(), e); } } + + + + /******************************************************************************* + ** Get the variant id from the session for the backend. + *******************************************************************************/ + protected Serializable getVariantId(QBackendMetaData backendMetaData) throws QException + { + QSession session = QContext.getQSession(); + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) + { + throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); + } + Serializable variantId = session.getBackendVariants().get(variantTypeKey); + return variantId; + } + + + + /******************************************************************************* + ** For backends that use variants, look up the variant record (in theory, based + ** on an id in the session's backend variants map, then fetched from the backend's + ** variant options table. + *******************************************************************************/ + protected QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException + { + Serializable variantId = getVariantId(backendMetaData); + GetInput getInput = new GetInput(); + getInput.setShouldMaskPasswords(false); + getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + getInput.setPrimaryKey(variantId); + GetOutput getOutput = new GetAction().execute(getInput); + + QRecord record = getOutput.getRecord(); + if(record == null) + { + throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); + } + return record; + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index f50f1b4c..6c9c6bc5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -41,6 +41,9 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private String contentsFieldName; private String fileNameFieldName; + private String sizeFieldName; + private String createDateFieldName; + private String modifyDateFieldName; @@ -281,4 +284,97 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails } } + + /******************************************************************************* + ** Getter for sizeFieldName + *******************************************************************************/ + public String getSizeFieldName() + { + return (this.sizeFieldName); + } + + + + /******************************************************************************* + ** Setter for sizeFieldName + *******************************************************************************/ + public void setSizeFieldName(String sizeFieldName) + { + this.sizeFieldName = sizeFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for sizeFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withSizeFieldName(String sizeFieldName) + { + this.sizeFieldName = sizeFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDateFieldName + *******************************************************************************/ + public String getCreateDateFieldName() + { + return (this.createDateFieldName); + } + + + + /******************************************************************************* + ** Setter for createDateFieldName + *******************************************************************************/ + public void setCreateDateFieldName(String createDateFieldName) + { + this.createDateFieldName = createDateFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for createDateFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withCreateDateFieldName(String createDateFieldName) + { + this.createDateFieldName = createDateFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDateFieldName + *******************************************************************************/ + public String getModifyDateFieldName() + { + return (this.modifyDateFieldName); + } + + + + /******************************************************************************* + ** Setter for modifyDateFieldName + *******************************************************************************/ + public void setModifyDateFieldName(String modifyDateFieldName) + { + this.modifyDateFieldName = modifyDateFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDateFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withModifyDateFieldName(String modifyDateFieldName) + { + this.modifyDateFieldName = modifyDateFieldName; + return (this); + } + + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java index 19c1601e..4157ffb6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java @@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory; import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; /******************************************************************************* @@ -64,6 +70,7 @@ public class FilesystemTableMetaDataBuilder { case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails(); case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails(); + case SFTPBackendModule.BACKEND_TYPE -> new SFTPTableBackendDetails(); default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType()); }; @@ -72,12 +79,27 @@ public class FilesystemTableMetaDataBuilder .withIsHidden(true) .withBackendName(backend.getName()) .withPrimaryKeyField("fileName") - .withField(new QFieldMetaData("fileName", QFieldType.INTEGER)) - .withField(new QFieldMetaData("contents", QFieldType.STRING)) + + .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB) + .withIsHeavy(true) + .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "File Contents"))) + + .withSection(SectionFactory.defaultT1("fileName")) + .withSection(SectionFactory.defaultT2("contents", "size")) + .withSection(SectionFactory.defaultT3("createDate", "modifyDate")) + .withBackendDetails(tableBackendDetails .withCardinality(Cardinality.ONE) .withFileNameFieldName("fileName") .withContentsFieldName("contents") + .withSizeFieldName("size") + .withCreateDateFieldName("createDate") + .withModifyDateFieldName("modifyDate") .withBasePath(basePath) .withGlob(glob)); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index c9fb807f..5dde1ab6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -35,6 +35,8 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -61,6 +63,50 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Long getFileSize(File file) + { + return (file.length()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileCreateDate(File file) + { + try + { + Path path = file.toPath(); + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); + FileTime creationTime = attrs.creationTime(); + return creationTime.toInstant(); + } + catch(IOException e) + { + LOG.warn("Error getting file createDate", e, logPair("file", file)); + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileModifyDate(File file) + { + return Instant.ofEpochMilli(file.lastModified()); + } + + + /******************************************************************************* ** List the files for this table. *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java index 43867e99..f41ceb6b 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java @@ -63,7 +63,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -314,7 +313,7 @@ public class FilesystemImporterStep implements BackendStep /******************************************************************************* ** *******************************************************************************/ - private static void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException + private static void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws QException { if(removeFileAfterImport) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index 8a9a6272..05610fe6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; @@ -35,6 +36,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3CountAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; @@ -112,6 +114,17 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new S3CountAction(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 91383c4b..bc0d96d5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import java.io.IOException; import java.io.InputStream; +import java.time.Instant; import java.util.List; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; @@ -56,11 +57,44 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction Date: Fri, 14 Feb 2025 20:26:44 -0600 Subject: [PATCH 10/67] Initial add of sftp filesystem module --- qqq-backend-module-filesystem/pom.xml | 24 ++ .../filesystem/sftp/SFTPBackendModule.java | 170 ++++++++++ .../sftp/actions/AbstractSFTPAction.java | 298 ++++++++++++++++++ .../sftp/actions/SFTPCountAction.java | 45 +++ .../sftp/actions/SFTPDeleteAction.java | 60 ++++ .../sftp/actions/SFTPInsertAction.java | 45 +++ .../sftp/actions/SFTPQueryAction.java | 45 +++ .../sftp/actions/SFTPStorageAction.java | 157 +++++++++ .../sftp/actions/SFTPUpdateAction.java | 62 ++++ .../sftp/model/SFTPDirEntryWithPath.java | 33 ++ .../model/metadata/SFTPBackendMetaData.java | 198 ++++++++++++ .../metadata/SFTPBackendVariantSetting.java | 38 +++ .../metadata/SFTPTableBackendDetails.java | 45 +++ .../sftp/utils/SFTPOutputStream.java | 179 +++++++++++ .../backend/module/filesystem/TestUtils.java | 140 +++++++- .../module/filesystem/sftp/BaseSFTPTest.java | 139 ++++++++ .../sftp/actions/SFTPCountActionTest.java | 64 ++++ .../sftp/actions/SFTPDeleteActionTest.java | 48 +++ .../sftp/actions/SFTPInsertActionTest.java | 120 +++++++ .../sftp/actions/SFTPQueryActionTest.java | 105 ++++++ .../sftp/actions/SFTPStorageActionTest.java | 115 +++++++ .../sftp/actions/SFTPUpdateActionTest.java | 48 +++ .../src/test/resources/files/testfile.txt | 3 + 23 files changed, 2176 insertions(+), 5 deletions(-) create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java create mode 100644 qqq-backend-module-filesystem/src/test/resources/files/testfile.txt diff --git a/qqq-backend-module-filesystem/pom.xml b/qqq-backend-module-filesystem/pom.xml index 32cc274f..9344db4a 100644 --- a/qqq-backend-module-filesystem/pom.xml +++ b/qqq-backend-module-filesystem/pom.xml @@ -50,6 +50,17 @@ aws-java-sdk-s3 1.12.261 + + org.apache.sshd + sshd-sftp + 2.14.0 + + + org.apache.sshd + sshd-sftp + 2.14.0 + + cloud.localstack localstack-utils @@ -57,6 +68,19 @@ test + + org.testcontainers + testcontainers + 1.15.3 + test + + + net.java.dev.jna + jna + 5.7.0 + test + + org.apache.maven.plugins diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java new file mode 100644 index 00000000..17c447dd --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java @@ -0,0 +1,170 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.AbstractSFTPAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPCountAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPDeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPInsertAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPStorageAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPUpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with SFTP filesystems (as a client) + *******************************************************************************/ +public class SFTPBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface +{ + public static final String BACKEND_TYPE = "sftp"; + + static + { + QBackendModuleDispatcher.registerBackendModule(new SFTPBackendModule()); + } + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractSFTPAction()); + } + + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return (BACKEND_TYPE); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (SFTPBackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return (SFTPTableBackendDetails.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new SFTPQueryAction(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new SFTPCountAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new SFTPInsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new SFTPUpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new SFTPDeleteAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return new SFTPStorageAction(); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java new file mode 100644 index 00000000..80db887e --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -0,0 +1,298 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import org.apache.sshd.client.SshClient; +import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.sftp.client.SftpClient; +import org.apache.sshd.sftp.client.SftpClientFactory; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Base class for all SFTP filesystem actions + *******************************************************************************/ +public class AbstractSFTPAction extends AbstractBaseFilesystemAction +{ + private static final QLogger LOG = QLogger.getLogger(AbstractSFTPAction.class); + + private SshClient sshClient; + private ClientSession clientSession; + private SftpClient sftpClient; + + + + /******************************************************************************* + ** Set up the sftp utils object to be used for this action. + *******************************************************************************/ + @Override + public void preAction(QBackendMetaData backendMetaData) throws QException + { + super.preAction(backendMetaData); + + if(sftpClient != null) + { + LOG.debug("sftpClient object is already set - not re-setting it."); + return; + } + + try + { + SFTPBackendMetaData sftpBackendMetaData = getBackendMetaData(SFTPBackendMetaData.class, backendMetaData); + + this.sshClient = SshClient.setUpDefaultClient(); + sshClient.start(); + + String username = sftpBackendMetaData.getUsername(); + String password = sftpBackendMetaData.getPassword(); + String hostName = sftpBackendMetaData.getHostName(); + Integer port = sftpBackendMetaData.getPort(); + + if(backendMetaData.getUsesVariants()) + { + QRecord variantRecord = getVariantRecord(backendMetaData); + LOG.debug("Getting SFTP connection credentials from variant record", + logPair("tableName", backendMetaData.getBackendVariantsConfig().getOptionsTableName()), + logPair("id", variantRecord.getValue("id")), + logPair("name", variantRecord.getRecordLabel())); + Map fieldNameMap = backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap(); + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.USERNAME)) + { + username = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.USERNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PASSWORD)) + { + password = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.PASSWORD)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.HOSTNAME)) + { + hostName = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.HOSTNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PORT)) + { + port = variantRecord.getValueInteger(fieldNameMap.get(SFTPBackendVariantSetting.PORT)); + } + } + + this.clientSession = sshClient.connect(username, hostName, port).verify().getSession(); + clientSession.addPasswordIdentity(password); + clientSession.auth().verify(); + + this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); + } + catch(IOException e) + { + throw (new QException("Error setting up SFTP connection", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Long getFileSize(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getSize(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileCreateDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getCreateTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileModifyDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getModifyTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + { + try + { + String fullPath = getFullBasePath(table, backendBase); + List rs = new ArrayList<>(); + + for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) + { + if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename())) + { + continue; + } + + if(dirEntry.getAttributes().isDirectory()) + { + // todo - recursive?? + continue; + } + + // todo filter/glob + // todo skip + // todo limit + // todo order by + rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry)); + } + + return (rs); + } + catch(Exception e) + { + throw (new QException("Error listing files", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public InputStream readFile(SFTPDirEntryWithPath dirEntry) throws IOException + { + return (sftpClient.read(getFullPathForFile(dirEntry))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException + { + sftpClient.put(new ByteArrayInputStream(contents), path); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getFullPathForFile(SFTPDirEntryWithPath dirEntry) + { + return (dirEntry.path() + "/" + dirEntry.dirEntry().getFilename()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + throw (new QRuntimeException("Not yet implemented")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + throw (new QRuntimeException("Not yet implemented")); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected SftpClient getSftpClient(QBackendMetaData backend) throws QException + { + if(sftpClient == null) + { + preAction(backend); + } + + return (sftpClient); + } +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java new file mode 100644 index 00000000..f882662c --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountAction extends AbstractSFTPAction implements CountInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public CountOutput execute(CountInput countInput) throws QException + { + return (executeCount(countInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java new file mode 100644 index 00000000..365e3ec7 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteOutput execute(DeleteInput deleteInput) throws QException + { + throw new NotImplementedException("SFTP delete not implemented"); + /* + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java new file mode 100644 index 00000000..aa1c81dc --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertAction extends AbstractSFTPAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertOutput execute(InsertInput insertInput) throws QException + { + return (super.executeInsert(insertInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java new file mode 100644 index 00000000..5759c7ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPQueryAction extends AbstractSFTPAction implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryOutput execute(QueryInput queryInput) throws QException + { + return (super.executeQuery(queryInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java new file mode 100644 index 00000000..ddcb40f1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.utils.SFTPOutputStream; +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** (mass, streamed) storage action for sftp module + *******************************************************************************/ +public class SFTPStorageAction extends AbstractSFTPAction implements QStorageInterface +{ + + /******************************************************************************* + ** create an output stream in the storage backend - that can be written to, + ** for the purpose of inserting or writing a file into storage. + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + + SFTPOutputStream sftpOutputStream = new SFTPOutputStream(sftpClient, getFullPath(storageInput)); + return (sftpOutputStream); + } + catch(Exception e) + { + throw (new QException("Exception creating sftp output stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFullPath(StorageInput storageInput) throws QException + { + QTableMetaData table = storageInput.getTable(); + QBackendMetaData backend = storageInput.getBackend(); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference()); + + ///////////////////////////////////////////////////////////// + // s3 seems to do better w/o leading slashes, so, strip... // + ///////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = fullPath.substring(1); + } + + return fullPath; + } + + + + /******************************************************************************* + ** create an input stream in the storage backend - that can be read from, + ** for the purpose of getting or reading a file from storage. + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + InputStream inputStream = sftpClient.read(getFullPath(storageInput)); + + return (inputStream); + } + catch(Exception e) + { + throw (new QException("Exception getting sftp input stream for file.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDownloadURL(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + //S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + //preAction(backend); + // + //AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + //String fullPath = getFullPath(storageInput); + //return (amazonS3.getUrl(backend.getBucketName(), fullPath).toString()); + } + catch(Exception e) + { + throw (new QException("Exception getting the sftp download URL.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void makePublic(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + } + catch(Exception e) + { + throw (new QException("Exception making sftp file publicly available.", e)); + } + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java new file mode 100644 index 00000000..93b31a69 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateOutput execute(UpdateInput updateInput) throws QException + { + throw new NotImplementedException("SFTP update not implemented"); + /* + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List records = new ArrayList<>(); + rs.setRecords(records); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java new file mode 100644 index 00000000..06cae267 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model; + + +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** + *******************************************************************************/ +public record SFTPDirEntryWithPath(String path, SftpClient.DirEntry dirEntry) +{ +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java new file mode 100644 index 00000000..c423f99e --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java @@ -0,0 +1,198 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP backend meta data. + *******************************************************************************/ +public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData +{ + private String username; + private String password; + private String hostName; + private Integer port; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPBackendMetaData() + { + super(); + setBackendType(SFTPBackendModule.class); + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public SFTPBackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public SFTPBackendMetaData withName(String name) + { + setName(name); + return this; + } + + + + /******************************************************************************* + ** Getter for username + *******************************************************************************/ + public String getUsername() + { + return (this.username); + } + + + + /******************************************************************************* + ** Setter for username + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + *******************************************************************************/ + public SFTPBackendMetaData withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + *******************************************************************************/ + public String getPassword() + { + return (this.password); + } + + + + /******************************************************************************* + ** Setter for password + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent setter for password + *******************************************************************************/ + public SFTPBackendMetaData withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Getter for hostName + *******************************************************************************/ + public String getHostName() + { + return (this.hostName); + } + + + + /******************************************************************************* + ** Setter for hostName + *******************************************************************************/ + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + + + /******************************************************************************* + ** Fluent setter for hostName + *******************************************************************************/ + public SFTPBackendMetaData withHostName(String hostName) + { + this.hostName = hostName; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + *******************************************************************************/ + public Integer getPort() + { + return (this.port); + } + + + + /******************************************************************************* + ** Setter for port + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent setter for port + *******************************************************************************/ + public SFTPBackendMetaData withPort(Integer port) + { + this.port = port; + return (this); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java new file mode 100644 index 00000000..b5205cb9 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum SFTPBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + HOSTNAME, + PORT, + BASE_PATH +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java new file mode 100644 index 00000000..3a1605e0 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP specific Extension of QTableBackendDetails + *******************************************************************************/ +public class SFTPTableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPTableBackendDetails() + { + super(); + setBackendType(SFTPBackendModule.class); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java new file mode 100644 index 00000000..19887c2a --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java @@ -0,0 +1,179 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.utils; + + +import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.sshd.sftp.client.SftpClient; +import org.jetbrains.annotations.NotNull; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPOutputStream extends PipedOutputStream +{ + private static final QLogger LOG = QLogger.getLogger(SFTPOutputStream.class); + + private final SftpClient sftpClient; + + private final PipedInputStream pipedInputStream; + private final Future putFuture; + + private AtomicBoolean started = new AtomicBoolean(false); + private AtomicReference putException = new AtomicReference<>(null); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public SFTPOutputStream(SftpClient sftpClient, String path) throws IOException + { + pipedInputStream = new PipedInputStream(this, 32 * 1024); + + this.sftpClient = sftpClient; + + putFuture = Executors.newSingleThreadExecutor().submit(() -> + { + try + { + started.set(true); + sftpClient.put(pipedInputStream, path); + } + catch(Exception e) + { + putException.set(e); + LOG.error("Error putting SFTP output stream", e); + + try + { + pipedInputStream.close(); + } + catch(IOException ex) + { + LOG.error("Secondary error closing pipedInputStream after sftp put error", e); + } + + throw new RuntimeException(e); + } + }); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void write(@NotNull byte[] b) throws IOException + { + try + { + super.write(b); + } + catch(IOException e) + { + if(putException.get() != null) + { + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error writing to SFTP output stream", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws IOException + { + try + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // don't try to close anything until we know that the sftpClient.put call's thread // + // has tried to start (otherwise, race condition could cause us to close things too early) // + ///////////////////////////////////////////////////////////////////////////////////////////// + int sleepLoops = 0; + while(!started.get() && sleepLoops++ <= 30) + { + SleepUtils.sleep(1, TimeUnit.SECONDS); + } + + /////////////////////////////////////////////////////////////////////////////////////// + // closing the pipedOutputStream (super) causes things to flush and complete the put // + /////////////////////////////////////////////////////////////////////////////////////// + super.close(); + + //////////////////////////////// + // wait for the put to finish // + //////////////////////////////// + putFuture.get(60 - sleepLoops, TimeUnit.SECONDS); + + /////////////////////////////////////////////////////////////////////////////// + // in case the put-future never did start, throw explicitly mentioning that. // + /////////////////////////////////////////////////////////////////////////////// + if(sleepLoops >= 30) + { + throw (new Exception("future to can sftpClient.put() was never started.")); + } + } + catch(ExecutionException ee) + { + throw new IOException("Error performing SFTP put", ee); + } + catch(Exception e) + { + if(putException.get() != null) + { + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error closing SFTP output stream", e); + } + finally + { + try + { + sftpClient.close(); + } + catch(IOException e) + { + LOG.error("Error closing SFTP client", e); + } + } + } + +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index acad9100..233cad0d 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem; import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.FilesystemTableMetaDataBuilder; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; @@ -52,6 +55,10 @@ import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fil import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; import org.apache.commons.io.FileUtils; @@ -60,20 +67,26 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; - public static final String BACKEND_NAME_S3 = "s3"; - public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; - public static final String BACKEND_NAME_MOCK = "mock"; - public static final String BACKEND_NAME_MEMORY = "memory"; + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; + public static final String BACKEND_NAME_SFTP = "sftp"; + public static final String BACKEND_NAME_SFTP_WITH_VARIANTS = "sftpWithVariants"; + public static final String BACKEND_NAME_MOCK = "mock"; + public static final String BACKEND_NAME_MEMORY = "memory"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String TABLE_NAME_PERSON_SFTP = "person-sftp"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; + public static final String TABLE_NAME_SFTP_FILE = "sftp-file"; + public static final String TABLE_NAME_SFTP_FILE_VARIANTS = "sftp-file-with-variants"; + public static final String TABLE_NAME_VARIANT_OPTIONS = "variant-options-table"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter"; @@ -148,6 +161,7 @@ public class TestUtils qInstance.addBackend(defineS3Backend()); qInstance.addBackend(defineS3BackendSansPrefix()); qInstance.addTable(defineS3CSVPersonTable()); + qInstance.addTable(defineSFTPCSVPersonTable()); qInstance.addTable(defineS3BlobTable()); qInstance.addTable(defineS3BlobSansPrefixTable()); qInstance.addBackend(defineMockBackend()); @@ -155,6 +169,15 @@ public class TestUtils qInstance.addTable(defineMockPersonTable()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); + QBackendMetaData sftpBackend = defineSFTPBackend(); + qInstance.addBackend(sftpBackend); + qInstance.addTable(defineSFTPFileTable(sftpBackend)); + + QBackendMetaData sftpBackendWithVariants = defineSFTPBackendWithVariants(); + qInstance.addBackend(sftpBackendWithVariants); + qInstance.addTable(defineSFTPFileTableWithVariants(sftpBackendWithVariants)); + qInstance.addTable(defineVariantOptionsTable()); + definePersonCsvImporter(qInstance); return (qInstance); @@ -162,6 +185,21 @@ public class TestUtils + /*************************************************************************** + ** + ***************************************************************************/ + private static QTableMetaData defineVariantOptionsTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_VARIANT_OPTIONS) + .withBackendName(defineMemoryBackend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("basePath", QFieldType.STRING)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -379,6 +417,25 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineSFTPCSVPersonTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON_SFTP) + .withLabel("Person SFTP Table") + .withBackendName(BACKEND_NAME_SFTP) + .withPrimaryKeyField("id") + .withFields(defineCommonPersonTableFields()) + .withBackendDetails(new SFTPTableBackendDetails() + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) + ); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -463,4 +520,77 @@ public class TestUtils MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); return (mockAuthenticationModule.createSession(defineInstance(), null)); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTable(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackend() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + .withBasePath(BaseSFTPTest.BACKEND_FOLDER) + .withName(BACKEND_NAME_SFTP)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTableWithVariants(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE_VARIANTS) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackendWithVariants() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + + //////////////////////////////////// + // only get basePath from variant // + //////////////////////////////////// + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withOptionsTableName(TABLE_NAME_VARIANT_OPTIONS) + .withVariantTypeKey(TABLE_NAME_VARIANT_OPTIONS) + .withBackendSettingSourceFieldNameMap(Map.of( + SFTPBackendVariantSetting.BASE_PATH, "basePath" + )) + ) + .withName(BACKEND_NAME_SFTP_WITH_VARIANTS)); + } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java new file mode 100644 index 00000000..69aa1e9e --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java @@ -0,0 +1,139 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.module.filesystem.BaseTest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.MountableFile; + + +/******************************************************************************* + ** Base class for tests that want to be able to work with sftp testcontainer + *******************************************************************************/ +public class BaseSFTPTest extends BaseTest +{ + public static final int PORT = 22; + public static final String USERNAME = "testuser"; + public static final String PASSWORD = "testpass"; + public static final String HOST_NAME = "localhost"; + + public static final String BACKEND_FOLDER = "upload"; + public static final String TABLE_FOLDER = "files"; + public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER; + + private static GenericContainer sftpContainer; + private static Integer currentPort; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @BeforeAll + static void setUp() throws Exception + { + sftpContainer = new GenericContainer<>("atmoz/sftp:latest") + .withExposedPorts(PORT) + .withCommand(USERNAME + ":" + PASSWORD + ":1001"); + + sftpContainer.start(); + + for(int i = 0; i < 5; i++) + { + sftpContainer.copyFileToContainer(MountableFile.forClasspathResource("files/testfile.txt"), REMOTE_DIR + "/testfile-" + i + ".txt"); + } + + grantUploadFilesDirWritePermission(); + + currentPort = sftpContainer.getMappedPort(22); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @AfterAll + static void tearDown() + { + if(sftpContainer != null) + { + sftpContainer.stop(); + } + } + + + + /******************************************************************************* + ** Getter for currentPort + ** + *******************************************************************************/ + public static Integer getCurrentPort() + { + return currentPort; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void revokeUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("444"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void grantUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("777"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void setUploadFilesDirPermission(String mode) throws Exception + { + sftpContainer.execInContainer("chmod", mode, "/home/testuser/upload/files"); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception + { + Container.ExecResult mkdir = sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path); + System.out.println(mkdir.getExitCode()); + } +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java new file mode 100644 index 00000000..643fe2ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java @@ -0,0 +1,64 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCount1() throws QException + { + CountInput countInput = initCountRequest(); + SFTPCountAction countAction = new SFTPCountAction(); + CountOutput countOutput = countAction.execute(countInput); + Assertions.assertEquals(5, countOutput.getCount(), "Expected # of rows from unfiltered count"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private CountInput initCountRequest() throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + return countInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java new file mode 100644 index 00000000..3a992fbf --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + assertThrows(NotImplementedException.class, () -> new SFTPDeleteAction().execute(new DeleteInput())); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java new file mode 100644 index 00000000..c648021c --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java @@ -0,0 +1,120 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.IOException; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOne() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + assertThat(insertOutput.getRecords()) + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseSFTPTest.BACKEND_FOLDER)); + + QRecord record = insertOutput.getRecords().get(0); + String fullPath = record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH); + assertThat(record.getErrors()).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOnePermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + + QRecord record = insertOutput.getRecords().get(0); + assertThat(record.getErrors()).isNotEmpty(); + assertThat(record.getErrors().get(0).getMessage()).contains("Error writing file: Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityMany() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_SFTP); + insertInput.setRecords(List.of( + new QRecord().withValue("id", "1").withValue("firstName", "Bob") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + assertThatThrownBy(() -> insertAction.execute(insertInput)) + .hasRootCauseInstanceOf(NotImplementedException.class); + } +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java new file mode 100644 index 00000000..fdce8969 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Unit test for SFTPQueryAction + *******************************************************************************/ +class SFTPQueryActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQuery1() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQueryVariantsTable() throws Exception + { + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_VARIANT_OPTIONS).withRecords(List.of( + new QRecord().withValue("id", 1).withValue("basePath", BaseSFTPTest.BACKEND_FOLDER), + new QRecord().withValue("id", 2).withValue("basePath", "empty-folder"), + new QRecord().withValue("id", 3).withValue("basePath", "non-existing-path") + ))); + + mkdirInSftpContainerUnderHomeTestuser("empty-folder/files"); + + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .hasMessageContaining("Could not find Backend Variant information for Backend"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 2)); + queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(0, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 3)); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .rootCause() + .hasMessageContaining("No such file"); + + // Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryInput initQueryRequest() throws QException + { + QueryInput queryInput = new QueryInput(); + return queryInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java new file mode 100644 index 00000000..ed49cc2d --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FilesystemStorageAction + *******************************************************************************/ +public class SFTPStorageActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSmall() throws Exception + { + String data = "Hellooo, Storage."; + runTest(data); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testPermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + String data = "oops!"; + assertThatThrownBy(() -> runTest(data)) + .hasRootCauseInstanceOf(IOException.class) + .rootCause() + .hasMessageContaining("Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLarge() throws Exception + { + String data = StringUtils.join("!", Collections.nCopies(5_000_000, "Hello")); + runTest(data); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void runTest(String data) throws QException, IOException + { + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_SFTP_FILE).withReference("fromStorageAction.txt"); + + StorageAction storageAction = new StorageAction(); + OutputStream outputStream = storageAction.createOutputStream(storageInput); + outputStream.write(data.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + InputStream inputStream = storageAction.getInputStream(storageInput); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + inputStream.transferTo(byteArrayOutputStream); + + assertEquals(data.length(), byteArrayOutputStream.toString(StandardCharsets.UTF_8).length()); + assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8)); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java new file mode 100644 index 00000000..2f025320 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + assertThrows(NotImplementedException.class, () -> new SFTPUpdateAction().execute(new UpdateInput())); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt new file mode 100644 index 00000000..eab448b1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt @@ -0,0 +1,3 @@ +This is a file. + +It is a test. From c341708d21327b28878f017f927820e7eed0ed69 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:30:13 -0600 Subject: [PATCH 11/67] Start (mostly done?) support for headless bulk-load --- .../BulkInsertReceiveFileMappingStep.java | 72 ++++++++++++++----- .../bulk/insert/BulkInsertStepUtils.java | 39 +++++++++- 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index afe3c9f1..94032fe5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -32,16 +32,20 @@ import java.util.Set; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.BooleanUtils; @@ -63,12 +67,37 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep { try { - BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + QRecord savedBulkLoadProfileRecord = BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); - /////////////////////////////////////////////////////////////////// - // read process values - construct a bulkLoadProfile out of them // - /////////////////////////////////////////////////////////////////// - BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + BulkLoadProfile bulkLoadProfile; + if(BulkInsertStepUtils.isHeadless(runBackendStepInput)) + { + ////////////////////////////////////////////////////////////////////////////// + // if running headless, build bulkLoadProfile from the saved profile record // + ////////////////////////////////////////////////////////////////////////////// + if(savedBulkLoadProfileRecord == null) + { + throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load")); + } + + SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord); + + try + { + bulkLoadProfile = JsonUtils.toObject(savedBulkLoadProfile.getMappingJson(), BulkLoadProfile.class); + } + catch(Exception e) + { + throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e)); + } + } + else + { + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + } ///////////////////////////////////////////////////////////////////////// // put the list of bulk load profile into the process state - it's the // @@ -183,21 +212,32 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep ///////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); - if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + if(BulkInsertStepUtils.isHeadless(runBackendStepInput)) { - ////////////////////////////////////////////////////////////////////////////////// - // just go to the prepareValueMapping backend step - it'll figure out the rest. // - // it's also where the value-mapping loop of steps points. // - // and, this will actually be the default (e.g., the step after this one). // - ////////////////////////////////////////////////////////////////////////////////// - runBackendStepInput.addValue("valueMappingFieldIndex", -1); + //////////////////////////////////////////////////////////////////////// + // if running headless, always go straight to the preview screen next // + // todo actually, we could make this execute, right? // + //////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); } else { - ////////////////////////////////////////////////////////////////////////////////// - // else - if no values to map - continue with the standard streamed-ETL preview // - ////////////////////////////////////////////////////////////////////////////////// - BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + { + ////////////////////////////////////////////////////////////////////////////////// + // just go to the prepareValueMapping backend step - it'll figure out the rest. // + // it's also where the value-mapping loop of steps points. // + // and, this will actually be the default (e.g., the step after this one). // + ////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.addValue("valueMappingFieldIndex", -1); + } + else + { + ////////////////////////////////////////////////////////////////////////////////// + // else - if no values to map - continue with the standard streamed-ETL preview // + ////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + } } } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index b0db3ad0..c52cd299 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; @@ -69,6 +70,18 @@ public class BulkInsertStepUtils + /*************************************************************************** + ** + ***************************************************************************/ + public static void setStorageInputForTheFile(RunProcessInput runProcessInput, StorageInput storageInput) + { + ArrayList storageInputs = new ArrayList<>(); + storageInputs.add(storageInput); + runProcessInput.addValue("theFile", storageInputs); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -144,13 +157,37 @@ public class BulkInsertStepUtils /*************************************************************************** ** ***************************************************************************/ - public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public static QRecord handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId"); if(savedBulkLoadProfileId != null) { QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId); runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord); + return (savedBulkLoadProfileRecord); } + + return (null); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static boolean isHeadless(RunBackendStepInput runBackendStepInput) + { + return (runBackendStepInput.getValuePrimitiveBoolean("isHeadless")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setHeadless(RunProcessInput runProcessInput) + { + runProcessInput.addValue("isHeadless", true); + } + } From 3f8c2957d1625723bbcfccccc833b0472157bde8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:45:51 -0600 Subject: [PATCH 12/67] Swap setVariantOptionsTableTypeField for setVariantOptionsTableTypeValue re: which one sets the new config's setVariantTypeKey --- .../qqq/backend/core/model/metadata/QBackendMetaData.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index 3d79e591..9bc7ad04 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -452,8 +452,6 @@ public class QBackendMetaData implements TopLevelMetaDataInterface @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") public void setVariantOptionsTableTypeField(String variantOptionsTableTypeField) { - this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeField); - this.variantOptionsTableTypeField = variantOptionsTableTypeField; if(this.variantOptionsTableTypeValue != null) { @@ -481,6 +479,8 @@ public class QBackendMetaData implements TopLevelMetaDataInterface @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") public void setVariantOptionsTableTypeValue(String variantOptionsTableTypeValue) { + this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeValue); + this.variantOptionsTableTypeValue = variantOptionsTableTypeValue; if(this.variantOptionsTableTypeField != null) { From be6d1b888ff87734f83b95580ab572bb4f6f69fb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:07:56 -0600 Subject: [PATCH 13/67] Add urlencoding to blob download urls --- .../backend/core/actions/values/QValueFormatter.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 39a6a520..6129b409 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -32,6 +34,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -532,7 +535,7 @@ public class QValueFormatter //////////////////////////////////////////////////////////////////////////////////////////////// // if field type is blob OR if there's a supplemental process or code-ref that needs to run - // - // then update its value to be a callback-url that'll give access to the bytes to download // + // then update its value to be a callback-url that'll give access to the bytes to download. // // implied here is that a String value (w/o supplemental code/proc) has its value stay as a // // URL, which is where the file is directly downloaded from. And in the case of a String // // with code-to-run, then the code should run, followed by a redirect to the value URL. // @@ -541,7 +544,10 @@ public class QValueFormatter || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE) || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME)) { - record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName); + record.setValue(field.getName(), "/data/" + table.getName() + "/" + + URLEncoder.encode(ValueUtils.getValueAsString(primaryKey), StandardCharsets.UTF_8) + "/" + + field.getName() + "/" + + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8)); } record.setDisplayValue(field.getName(), fileName); } From 8816177df8dd793f67cdaec858af86303988ff37 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:49:33 -0600 Subject: [PATCH 14/67] Add optional variantRecordLookupFunction to BackendVariantsConfig and validation of same; refactor up some shared backend code into BackendVariantsUtil --- .../core/instances/QInstanceValidator.java | 52 ++++++--- .../variants/BackendVariantsConfig.java | 40 ++++++- .../variants/BackendVariantsUtil.java | 106 ++++++++++++++++++ .../instances/QInstanceValidatorTest.java | 76 ++++++++++++- .../variants/BackendVariantsUtilTest.java | 100 +++++++++++++++++ 5 files changed, 358 insertions(+), 16 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index fa4d1a23..a214ee6b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -37,7 +37,9 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TimeZone; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -115,6 +117,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; import org.apache.commons.lang.BooleanUtils; import org.quartz.CronExpression; @@ -556,8 +559,8 @@ public class QInstanceValidator { assertCondition(StringUtils.hasContent(backendVariantsConfig.getVariantTypeKey()), "Missing variantTypeKey in backendVariantsConfig in [" + backendName + "]"); - String optionsTableName = backendVariantsConfig.getOptionsTableName(); - QTableMetaData optionsTable = qInstance.getTable(optionsTableName); + String optionsTableName = backendVariantsConfig.getOptionsTableName(); + QTableMetaData optionsTable = qInstance.getTable(optionsTableName); if(assertCondition(StringUtils.hasContent(optionsTableName), "Missing optionsTableName in backendVariantsConfig in [" + backendName + "]")) { if(assertCondition(optionsTable != null, "Unrecognized optionsTableName [" + optionsTableName + "] in backendVariantsConfig in [" + backendName + "]")) @@ -573,7 +576,11 @@ public class QInstanceValidator Map backendSettingSourceFieldNameMap = backendVariantsConfig.getBackendSettingSourceFieldNameMap(); if(assertCondition(CollectionUtils.nullSafeHasContents(backendSettingSourceFieldNameMap), "Missing or empty backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]")) { - if(optionsTable != null) + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // only validate field names in the backendSettingSourceFieldNameMap if there is NOT a variantRecordSupplier // + // (the idea being, that the supplier might be building a record with fieldNames that aren't in the table... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(optionsTable != null && backendVariantsConfig.getVariantRecordLookupFunction() == null) { for(Map.Entry entry : backendSettingSourceFieldNameMap.entrySet()) { @@ -581,6 +588,11 @@ public class QInstanceValidator } } } + + if(backendVariantsConfig.getVariantRecordLookupFunction() != null) + { + validateSimpleCodeReference("VariantRecordSupplier in backendVariantsConfig in backend [" + backendName + "]: ", backendVariantsConfig.getVariantRecordLookupFunction(), UnsafeFunction.class, Function.class); + } } } else @@ -1404,7 +1416,7 @@ public class QInstanceValidator //////////////////////////////////////////////////////////////////////// if(customizerInstance != null && tableCustomizer.getExpectedType() != null) { - assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance); + assertObjectCanBeCasted(prefix, customizerInstance, tableCustomizer.getExpectedType()); } } } @@ -1416,18 +1428,31 @@ public class QInstanceValidator /******************************************************************************* ** Make sure that a given object can be casted to an expected type. *******************************************************************************/ - private T assertObjectCanBeCasted(String errorPrefix, Class expectedType, Object object) + private void assertObjectCanBeCasted(String errorPrefix, Object object, Class... anyOfExpectedClasses) { - T castedObject = null; - try + for(Class expectedClass : anyOfExpectedClasses) { - castedObject = expectedType.cast(object); + try + { + expectedClass.cast(object); + return; + } + catch(ClassCastException e) + { + ///////////////////////////////////// + // try next type (if there is one) // + ///////////////////////////////////// + } } - catch(ClassCastException e) + + if(anyOfExpectedClasses.length == 1) { - errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType); + errors.add(errorPrefix + "CodeReference is not of the expected type: " + anyOfExpectedClasses[0]); + } + else + { + errors.add(errorPrefix + "CodeReference is not any of the expected types: " + Arrays.stream(anyOfExpectedClasses).map(c -> c.getName()).collect(Collectors.joining(", "))); } - return castedObject; } @@ -2171,7 +2196,8 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class expectedClass) + @SafeVarargs + private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class... anyOfExpectedClasses) { if(!preAssertionsForCodeReference(codeReference, prefix)) { @@ -2199,7 +2225,7 @@ public class QInstanceValidator //////////////////////////////////////////////////////////////////////// if(classInstance != null) { - assertObjectCanBeCasted(prefix, expectedClass, classInstance); + assertObjectCanBeCasted(prefix, classInstance, anyOfExpectedClasses); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java index 20237e7a..290c099f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.variants; import java.util.HashMap; import java.util.Map; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; /******************************************************************************* @@ -37,13 +38,17 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; ** field names in that table that they come from. e.g., a backend may have a ** username attribute, whose value comes from a field named "theUser" in the ** variant options table. + ** - an optional code reference to a variantRecordLookupFunction - to customize + ** how the variant record is looked up (such as, adding joined or other custom + ** fields). *******************************************************************************/ public class BackendVariantsConfig { private String variantTypeKey; - private String optionsTableName; - private QQueryFilter optionsFilter; + private String optionsTableName; + private QQueryFilter optionsFilter; + private QCodeReference variantRecordLookupFunction; private Map backendSettingSourceFieldNameMap; @@ -186,4 +191,35 @@ public class BackendVariantsConfig return (this); } + + /******************************************************************************* + ** Getter for variantRecordLookupFunction + *******************************************************************************/ + public QCodeReference getVariantRecordLookupFunction() + { + return (this.variantRecordLookupFunction); + } + + + + /******************************************************************************* + ** Setter for variantRecordLookupFunction + *******************************************************************************/ + public void setVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction) + { + this.variantRecordLookupFunction = variantRecordLookupFunction; + } + + + + /******************************************************************************* + ** Fluent setter for variantRecordLookupFunction + *******************************************************************************/ + public BackendVariantsConfig withVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction) + { + this.variantRecordLookupFunction = variantRecordLookupFunction; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java new file mode 100644 index 00000000..a94fbc4e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.io.Serializable; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; + + +/******************************************************************************* + ** Utility methods for backends working with Variants. + *******************************************************************************/ +public class BackendVariantsUtil +{ + + /******************************************************************************* + ** Get the variant id from the session for the backend. + *******************************************************************************/ + public static Serializable getVariantId(QBackendMetaData backendMetaData) throws QException + { + QSession session = QContext.getQSession(); + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) + { + throw (new QException("Could not find Backend Variant information in session under key '" + variantTypeKey + "' for Backend '" + backendMetaData.getName() + "'")); + } + Serializable variantId = session.getBackendVariants().get(variantTypeKey); + return variantId; + } + + + + /******************************************************************************* + ** For backends that use variants, look up the variant record (in theory, based + ** on an id in the session's backend variants map, then fetched from the backend's + ** variant options table. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException + { + Serializable variantId = getVariantId(backendMetaData); + + QRecord record; + if(backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction() != null) + { + Object o = QCodeLoader.getAdHoc(Object.class, backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction()); + if(o instanceof UnsafeFunction unsafeFunction) + { + record = ((UnsafeFunction) unsafeFunction).apply(variantId); + } + else if(o instanceof Function function) + { + record = ((Function) function).apply(variantId); + } + else + { + throw (new QException("Backend Variant's recordLookupFunction is not of any expected type (should have been caught by instance validation??)")); + } + } + else + { + GetInput getInput = new GetInput(); + getInput.setShouldMaskPasswords(false); + getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + getInput.setPrimaryKey(variantId); + GetOutput getOutput = new GetAction().execute(getInput); + + record = getOutput.getRecord(); + } + + if(record == null) + { + throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); + } + return record; + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 9ce5d9f6..8419b05a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -103,6 +103,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -246,6 +247,79 @@ public class QInstanceValidatorTest extends BaseTest .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) .withBackendSettingSourceFieldNameMap(Map.of(setting, "noSuchField")))), "Unrecognized fieldName [noSuchField] in backendSettingSourceFieldNameMap"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(CustomizerThatIsNotOfTheRightBaseClass.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + )), + "VariantRecordSupplier in backendVariantsConfig in backend [variant]: CodeReference is not any of the expected types: com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction, java.util.function.Function"); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")) + ))); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(VariantRecordFunction.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + ))); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(VariantRecordUnsafeFunction.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + ))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class VariantRecordFunction implements Function + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QRecord apply(Serializable serializable) + { + return null; + } + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class VariantRecordUnsafeFunction implements UnsafeFunction + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QRecord apply(Serializable serializable) throws QException + { + return null; + } } @@ -2437,7 +2511,7 @@ public class QInstanceValidatorTest extends BaseTest { int noOfReasons = actualReasons == null ? 0 : actualReasons.size(); assertEquals(expectedReasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", expectedReasons) - + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--")); + + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--")); } for(String reason : expectedReasons) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java new file mode 100644 index 00000000..5957ee61 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java @@ -0,0 +1,100 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for BackendVariantsUtil + *******************************************************************************/ +class BackendVariantsUtilTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetVariantId() throws QException + { + QBackendMetaData myBackend = getBackendMetaData(); + + assertThatThrownBy(() -> BackendVariantsUtil.getVariantId(myBackend)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701)); + assertEquals(1701, BackendVariantsUtil.getVariantId(myBackend)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData getBackendMetaData() + { + QBackendMetaData myBackend = new QBackendMetaData() + .withName("TestBackend") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withOptionsTableName(TestUtils.TABLE_NAME_SHAPE) + .withVariantTypeKey("yourSelectedShape")); + return myBackend; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetVariantRecord() throws QException + { + QBackendMetaData myBackend = getBackendMetaData(); + + TestUtils.insertDefaultShapes(QContext.getQInstance()); + + assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701)); + assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend)) + .hasMessageContaining("Could not find Backend Variant in table shape with id '1701'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1)); + QRecord variantRecord = BackendVariantsUtil.getVariantRecord(myBackend); + assertEquals(1, variantRecord.getValueInteger("id")); + assertNotNull(variantRecord.getValue("name")); + } + +} \ No newline at end of file From 143ed927fa845009cd1e9d3fe6a97c6fadfa1cf2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:50:27 -0600 Subject: [PATCH 15/67] add ability to set and trace processTracerKeyRecord in bulk load --- .../processes/RunBackendStepInput.java | 8 ++- .../bulk/insert/BulkInsertExtractStep.java | 2 + .../bulk/insert/BulkInsertStepUtils.java | 26 ++++++++ .../ProcessTracerKeyRecordMessage.java | 66 +++++++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 764f930c..84da02f7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -257,7 +257,11 @@ public class RunBackendStepInput extends AbstractActionInput public List getRecordsAsEntities(Class entityClass) throws QException { List rs = new ArrayList<>(); - for(QRecord record : processState.getRecords()) + + /////////////////////////////////////////////////////////////////////////////////// + // note - important to call getRecords here, which is overwritten in subclasses! // + /////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : getRecords()) { rs.add(QRecordEntity.fromQRecord(entityClass, record)); } @@ -601,7 +605,7 @@ public class RunBackendStepInput extends AbstractActionInput ***************************************************************************/ public void traceMessage(ProcessTracerMessage message) { - if(processTracer != null) + if(processTracer != null && message != null) { try { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index 694ae332..dabfb3f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -55,6 +55,8 @@ public class BulkInsertExtractStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); + int rowsAdded = 0; int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index c52cd299..e9c09cea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerKeyRecordMessage; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.BooleanUtils; import org.json.JSONArray; @@ -190,4 +191,29 @@ public class BulkInsertStepUtils runProcessInput.addValue("isHeadless", true); } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setProcessTracerKeyRecordMessage(RunProcessInput runProcessInput, ProcessTracerKeyRecordMessage processTracerKeyRecordMessage) + { + runProcessInput.addValue("processTracerKeyRecordMessage", processTracerKeyRecordMessage); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProcessTracerKeyRecordMessage getProcessTracerKeyRecordMessage(RunBackendStepInput runBackendStepInput) + { + Serializable value = runBackendStepInput.getValue("processTracerKeyRecordMessage"); + if(value instanceof ProcessTracerKeyRecordMessage processTracerKeyRecordMessage) + { + return (processTracerKeyRecordMessage); + } + + return (null); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java new file mode 100644 index 00000000..53938092 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java @@ -0,0 +1,66 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +/******************************************************************************* + ** Specialization of process tracer message, to indicate a 'key record' that was + ** used as an input or trigger to a process. + *******************************************************************************/ +public class ProcessTracerKeyRecordMessage extends ProcessTracerMessage +{ + private final String tableName; + private final Integer recordId; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public ProcessTracerKeyRecordMessage(String tableName, Integer recordId) + { + super("Process Key Record is " + tableName + " " + recordId); + this.tableName = tableName; + this.recordId = recordId; + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Getter for recordId + *******************************************************************************/ + public Integer getRecordId() + { + return (this.recordId); + } + +} From 0005c51ecdbdfd6e88e3a61baa9f9fca456586af Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:50:47 -0600 Subject: [PATCH 16/67] Add capturing and reporting first & last inserted primary keys --- .../bulk/insert/BulkInsertLoadStep.java | 82 ++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java index c8b73ec2..b35bb24b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java @@ -22,16 +22,38 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* ** *******************************************************************************/ -public class BulkInsertLoadStep extends LoadViaInsertStep +public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSummaryProviderInterface { + private static final QLogger LOG = QLogger.getLogger(BulkInsertLoadStep.class); + + private Serializable firstInsertedPrimaryKey = null; + private Serializable lastInsertedPrimaryKey = null; + + /******************************************************************************* ** @@ -42,4 +64,62 @@ public class BulkInsertLoadStep extends LoadViaInsertStep return (QInputSource.USER); } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + super.runOnePage(runBackendStepInput, runBackendStepOutput); + + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName")); + + List insertedRecords = runBackendStepOutput.getRecords(); + for(QRecord insertedRecord : insertedRecords) + { + if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors())) + { + if(firstInsertedPrimaryKey == null) + { + firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + } + + lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + ArrayList processSummary = getTransformStep().getProcessSummary(runBackendStepOutput, isForResultScreen); + + try + { + if(firstInsertedPrimaryKey != null) + { + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepOutput.getValueString("tableName")); + QFieldMetaData field = table.getField(table.getPrimaryKeyField()); + if(field.getType().isNumeric()) + { + ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey); + idsLine.setCount(null); + processSummary.add(idsLine); + } + } + } + catch(Exception e) + { + LOG.warn("Error adding inserted-keys process summary line", e); + } + + return (processSummary); + } } From 2fd3ed256136fbde585c51825ff7c4c5176e5c58 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:51:05 -0600 Subject: [PATCH 17/67] add serializable --- .../backend/core/processes/tracing/ProcessTracerMessage.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java index df2dd558..bb27a0e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java @@ -22,13 +22,16 @@ package com.kingsrook.qqq.backend.core.processes.tracing; +import java.io.Serializable; + + /******************************************************************************* ** Basic class that can be passed in to ProcessTracerInterface.handleMessage. ** This class just provides for a string message. We anticipate subclasses ** that may have more specific data, that specific tracer implementations may ** be aware of. *******************************************************************************/ -public class ProcessTracerMessage +public class ProcessTracerMessage implements Serializable { private String message; From dc25f6b289aa69dd4b8b86de595c69da2d1a1f35 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:52:49 -0600 Subject: [PATCH 18/67] Explicit exception if table name is not given. --- .../qqq/backend/core/actions/tables/DeleteAction.java | 5 +++++ .../qqq/backend/core/actions/tables/InsertAction.java | 7 +++++++ .../qqq/backend/core/actions/tables/UpdateAction.java | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index 0ded3c93..a5028027 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -82,6 +82,11 @@ public class DeleteAction { ActionHelper.validateSession(deleteInput); + if(deleteInput.getTableName() == null) + { + throw (new QException("Table name was not specified in delete input")); + } + QTableMetaData table = deleteInput.getTable(); String primaryKeyFieldName = table.getPrimaryKeyField(); QFieldMetaData primaryKeyField = table.getField(primaryKeyFieldName); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 64a4ddc3..846ed476 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -110,6 +111,12 @@ public class InsertAction extends AbstractQActionFunction Date: Wed, 19 Feb 2025 19:53:14 -0600 Subject: [PATCH 19/67] Move variant lookups to new BackendVariantsUtil --- .../module/api/actions/BaseAPIActionUtil.java | 50 ++----------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 3623db11..2d34e264 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -36,7 +36,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -62,6 +61,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; @@ -779,7 +779,7 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); return (record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.API_KEY, APIBackendVariantSetting.API_KEY))); } @@ -807,7 +807,7 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); return (Pair.of( record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.USERNAME, APIBackendVariantSetting.USERNAME)), record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.PASSWORD, APIBackendVariantSetting.PASSWORD)) @@ -819,46 +819,6 @@ public class BaseAPIActionUtil - /******************************************************************************* - ** For backends that use variants, look up the variant record (in theory, based - ** on an id in the session's backend variants map, then fetched from the backend's - ** variant options table. - *******************************************************************************/ - protected QRecord getVariantRecord() throws QException - { - Serializable variantId = getVariantId(); - GetInput getInput = new GetInput(); - getInput.setShouldMaskPasswords(false); - getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); - getInput.setPrimaryKey(variantId); - GetOutput getOutput = new GetAction().execute(getInput); - - QRecord record = getOutput.getRecord(); - if(record == null) - { - throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); - } - return record; - } - - - - /******************************************************************************* - ** Get the variant id from the session for the backend. - *******************************************************************************/ - protected Serializable getVariantId() throws QException - { - QSession session = QContext.getQSession(); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getBackendVariantsConfig().getVariantTypeKey())) - { - throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); - } - Serializable variantId = session.getBackendVariants().get(backendMetaData.getBackendVariantsConfig().getVariantTypeKey()); - return variantId; - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -871,7 +831,7 @@ public class BaseAPIActionUtil String accessTokenKey = "accessToken"; if(backendMetaData.getUsesVariants()) { - Serializable variantId = getVariantId(); + Serializable variantId = BackendVariantsUtil.getVariantId(backendMetaData); accessTokenKey = accessTokenKey + ":" + variantId; } @@ -961,7 +921,7 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); return (Pair.of( record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_ID, APIBackendVariantSetting.CLIENT_ID)), record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_SECRET, APIBackendVariantSetting.CLIENT_SECRET)) From 154c5442af5e0149682b34936b3b00f8bf895901 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:54:37 -0600 Subject: [PATCH 20/67] Add postAction(); move variants stuff to new BackendVariantsUtil; add baseName to ONE records; remove path criteria when filtering (assuming the listFiles method did it) --- .../actions/AbstractBaseFilesystemAction.java | 210 +++++++++++------- 1 file changed, 130 insertions(+), 80 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index a896230c..0557e68a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -34,18 +34,16 @@ import java.util.List; import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; -import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -56,10 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; -import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; @@ -156,9 +156,10 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) { - String tablePath = getFullBasePath(table, backend); - String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); - return (strippedPath); + String tablePath = getFullBasePath(table, backend); + String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); + String withoutLeadingSlash = stripLeadingSlash(strippedPath); // todo - dangerous, do all backends really want this?? + return (withoutLeadingSlash); } @@ -211,6 +212,34 @@ public abstract class AbstractBaseFilesystemAction + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripLeadingSlash(String path) + { + if(path == null) + { + return (null); + } + return (path.replaceFirst("^/+", "")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripTrailingSlash(String path) + { + if(path == null) + { + return (null); + } + return (path.replaceFirst("/+$", "")); + } + + + /******************************************************************************* ** Get the backend metaData, type-checked as the requested type. *******************************************************************************/ @@ -269,9 +298,31 @@ public abstract class AbstractBaseFilesystemAction LOG.warn("Error executing query", e); throw new QException("Error executing query", e); } + finally + { + postAction(); + } } + /*************************************************************************** + ** + ***************************************************************************/ + private void setRecordValueIfFieldNameHasContent(QRecord record, String fieldName, UnsafeSupplier valueSupplier) + { + if(StringUtils.hasContent(fieldName)) + { + try + { + record.setValue(fieldName, valueSupplier.get()); + } + catch(Exception e) + { + LOG.warn("Error setting record value for field", e, logPair("fieldName", fieldName)); + } + } + } + /*************************************************************************** ** @@ -287,24 +338,15 @@ public abstract class AbstractBaseFilesystemAction // for one-record tables, put the entire file's contents into a single record // //////////////////////////////////////////////////////////////////////////////// String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); - QRecord record = new QRecord().withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase); + QRecord record = new QRecord(); - if(StringUtils.hasContent(tableDetails.getSizeFieldName())) - { - record.setValue(tableDetails.getSizeFieldName(), getFileSize(file)); - } + setRecordValueIfFieldNameHasContent(record, tableDetails.getFileNameFieldName(), () -> filePathWithoutBase); + setRecordValueIfFieldNameHasContent(record, tableDetails.getBaseNameFieldName(), () -> stripAllPaths(filePathWithoutBase)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getSizeFieldName(), () -> getFileSize(file)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getCreateDateFieldName(), () -> getFileCreateDate(file)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getModifyDateFieldName(), () -> getFileModifyDate(file)); - if(StringUtils.hasContent(tableDetails.getCreateDateFieldName())) - { - record.setValue(tableDetails.getCreateDateFieldName(), getFileCreateDate(file)); - } - - if(StringUtils.hasContent(tableDetails.getModifyDateFieldName())) - { - record.setValue(tableDetails.getModifyDateFieldName(), getFileModifyDate(file)); - } - - if(shouldFileContentsBeRead(queryInput, table, tableDetails)) + if(shouldHeavyFileContentsBeRead(queryInput, table, tableDetails)) { try(InputStream inputStream = readFile(file)) { @@ -318,27 +360,37 @@ public abstract class AbstractBaseFilesystemAction } else { - if(StringUtils.hasContent(tableDetails.getSizeFieldName())) + Long size = record.getValueLong(tableDetails.getSizeFieldName()); + if(size != null) { - Long size = record.getValueLong(tableDetails.getSizeFieldName()); - if(size != null) + if(record.getBackendDetails() == null) { - if(record.getBackendDetails() == null) - { - record.setBackendDetails(new HashMap<>()); - } - - if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null) - { - record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>()); - } - - ((Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size); + record.setBackendDetails(new HashMap<>()); } + + if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null) + { + record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>()); + } + + ((Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size); } } - if(BackendQueryFilterUtils.doesRecordMatch(queryInput.getFilter(), null, record)) + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // the listFiles method may have used a "path" criteria. // + // if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + QQueryFilter filterForRecords = queryInput.getFilter(); + if(filterForRecords != null) + { + filterForRecords = filterForRecords.clone(); + + CollectionUtils.nonNullList(filterForRecords.getCriteria()) + .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria); + } + + if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record)) { records.add(record); } @@ -351,6 +403,31 @@ public abstract class AbstractBaseFilesystemAction + /*************************************************************************** + ** + ***************************************************************************/ + private Serializable stripAllPaths(String filePath) + { + if(filePath == null) + { + return null; + } + + return (filePath.replaceFirst(".*/", "")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static boolean isPathEqualsCriteria(QFilterCriteria criteria) + { + return "path".equals(criteria.getFieldName()) && QCriteriaOperator.EQUALS.equals(criteria.getOperator()); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -409,7 +486,7 @@ public abstract class AbstractBaseFilesystemAction /*************************************************************************** ** ***************************************************************************/ - private static boolean shouldFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) + private static boolean shouldHeavyFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) { boolean doReadContents = true; if(table.getField(tableDetails.getContentsFieldName()).getIsHeavy()) @@ -478,11 +555,21 @@ public abstract class AbstractBaseFilesystemAction { if(backendMetaData.getUsesVariants()) { - this.backendVariantRecord = getVariantRecord(backendMetaData); + this.backendVariantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData); } } + /*************************************************************************** + ** Method that subclasses can override to add post-action things (e.g., closing resources) + ***************************************************************************/ + public void postAction() + { + ////////////////// + // noop in base // + ////////////////// + } + /******************************************************************************* ** @@ -558,47 +645,10 @@ public abstract class AbstractBaseFilesystemAction { throw new QException("Error executing insert: " + e.getMessage(), e); } - } - - - - /******************************************************************************* - ** Get the variant id from the session for the backend. - *******************************************************************************/ - protected Serializable getVariantId(QBackendMetaData backendMetaData) throws QException - { - QSession session = QContext.getQSession(); - String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) + finally { - throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); + postAction(); } - Serializable variantId = session.getBackendVariants().get(variantTypeKey); - return variantId; - } - - - - /******************************************************************************* - ** For backends that use variants, look up the variant record (in theory, based - ** on an id in the session's backend variants map, then fetched from the backend's - ** variant options table. - *******************************************************************************/ - protected QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException - { - Serializable variantId = getVariantId(backendMetaData); - GetInput getInput = new GetInput(); - getInput.setShouldMaskPasswords(false); - getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); - getInput.setPrimaryKey(variantId); - GetOutput getOutput = new GetAction().execute(getInput); - - QRecord record = getOutput.getRecord(); - if(record == null) - { - throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); - } - return record; } } From 91aa8faca2d5e07451857920e00fcf440b27a84e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:54:47 -0600 Subject: [PATCH 21/67] Add baseNameFieldName --- ...AbstractFilesystemTableBackendDetails.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index 6c9c6bc5..307849b6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -41,6 +41,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private String contentsFieldName; private String fileNameFieldName; + private String baseNameFieldName; private String sizeFieldName; private String createDateFieldName; private String modifyDateFieldName; @@ -377,4 +378,35 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails } + + /******************************************************************************* + ** Getter for baseNameFieldName + *******************************************************************************/ + public String getBaseNameFieldName() + { + return (this.baseNameFieldName); + } + + + + /******************************************************************************* + ** Setter for baseNameFieldName + *******************************************************************************/ + public void setBaseNameFieldName(String baseNameFieldName) + { + this.baseNameFieldName = baseNameFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for baseNameFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withBaseNameFieldName(String baseNameFieldName) + { + this.baseNameFieldName = baseNameFieldName; + return (this); + } + + } From 31a586f23ee853946530e029797ae2e72cff06dc Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:54:58 -0600 Subject: [PATCH 22/67] Move stripLeadingSlash up to base class --- .../filesystem/s3/actions/AbstractS3Action.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index bc0d96d5..4c5e7b8c 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -213,20 +213,6 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction Date: Wed, 19 Feb 2025 20:01:30 -0600 Subject: [PATCH 23/67] Move makeConnection to its own method (for use by test process); add postAction to try to close the things; add looking for 'path' criteria and adding it to readDir call --- .../sftp/actions/AbstractSFTPAction.java | 84 +++++++++++++++---- .../module/filesystem/sftp/BaseSFTPTest.java | 23 ++++- .../sftp/actions/SFTPQueryActionTest.java | 83 +++++++++++++++--- 3 files changed, 158 insertions(+), 32 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 80db887e..c9e3c8b5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -23,21 +23,26 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; @@ -81,9 +86,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction closer = closable -> + { + if(closable != null) + { + try + { + closable.close(); + } + catch(Exception e) + { + LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName())); + } + } + }; + + closer.accept(sshClient); + closer.accept(clientSession); + closer.accept(sftpClient); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected SftpClient makeConnection(String username, String hostName, Integer port, String password) throws IOException + { + this.sshClient = SshClient.setUpDefaultClient(); + sshClient.start(); + + this.clientSession = sshClient.connect(username, hostName, port).verify().getSession(); + clientSession.addPasswordIdentity(password); + clientSession.auth().verify(); + + this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); + return (this.sftpClient); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -195,8 +239,22 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction rs = new ArrayList<>(); + String fullPath = getFullBasePath(table, backendBase); + + // todo - move somewhere shared + // todo - should all do this? + if(filter != null) + { + for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria())) + { + if(isPathEqualsCriteria(criteria)) + { + fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar); + } + } + } + + List rs = new ArrayList<>(); for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) { @@ -211,10 +269,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction patternExpectedCountMap = Map.of( + "%.txt", 3, + "sub%", 2, + "%1%", 1, + "%", 3, + "*", 0 + ); + + for(Map.Entry entry : patternExpectedCountMap.entrySet()) + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")) + .withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey()))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey()); + } + } + finally + { + rmrfInContainer(subfolderPath); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -71,7 +139,7 @@ class SFTPQueryActionTest extends BaseSFTPTest mkdirInSftpContainerUnderHomeTestuser("empty-folder/files"); - QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); assertThatThrownBy(() -> new QueryAction().execute(queryInput)) .hasMessageContaining("Could not find Backend Variant information for Backend"); @@ -91,15 +159,4 @@ class SFTPQueryActionTest extends BaseSFTPTest // Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); } - - - /******************************************************************************* - ** - *******************************************************************************/ - private QueryInput initQueryRequest() throws QException - { - QueryInput queryInput = new QueryInput(); - return queryInput; - } - } \ No newline at end of file From bb1a43f11f0dfe016614deded7034d0fab4f2d99 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 20:02:05 -0600 Subject: [PATCH 24/67] Initial checkin --- .../actions/SFTPTestConnectionAction.java | 404 ++++++++++++++++++ .../actions/SFTPTestConnectionActionTest.java | 178 ++++++++ 2 files changed, 582 insertions(+) create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java new file mode 100644 index 00000000..53b93ccf --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java @@ -0,0 +1,404 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** action for testing credentials for an SFTP backend connection + *******************************************************************************/ +public class SFTPTestConnectionAction extends AbstractSFTPAction +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public SFTPTestConnectionTestOutput testConnection(SFTPTestConnectionTestInput input) + { + try(SftpClient sftpClient = super.makeConnection(input.getUsername(), input.getHostName(), input.getPort(), input.getPassword())) + { + SFTPTestConnectionTestOutput output = new SFTPTestConnectionTestOutput().withIsConnectionSuccess(true); + + if(StringUtils.hasContent(input.basePath)) + { + try + { + Iterable dirEntries = sftpClient.readDir(input.basePath); + + ///////////////////////////////////////////////////////////////////////// + // it seems like only the .iterator call throws if bad directory here. // + ///////////////////////////////////////////////////////////////////////// + dirEntries.iterator(); + output.setIsListBasePathSuccess(true); + } + catch(Exception e) + { + output.setIsListBasePathSuccess(false); + output.setListBasePathErrorMessage(e.getMessage()); + } + } + + return output; + } + catch(Exception e) + { + return new SFTPTestConnectionTestOutput().withIsConnectionSuccess(false).withConnectionErrorMessage(e.getMessage()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class SFTPTestConnectionTestInput + { + private String username; + private String hostName; + private Integer port; + private String password; + private String basePath; + + + + /******************************************************************************* + ** Getter for username + ** + *******************************************************************************/ + public String getUsername() + { + return username; + } + + + + /******************************************************************************* + ** Setter for username + ** + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for hostName + ** + *******************************************************************************/ + public String getHostName() + { + return hostName; + } + + + + /******************************************************************************* + ** Setter for hostName + ** + *******************************************************************************/ + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + + + /******************************************************************************* + ** Fluent setter for hostName + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withHostName(String hostName) + { + this.hostName = hostName; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + ** + *******************************************************************************/ + public Integer getPort() + { + return port; + } + + + + /******************************************************************************* + ** Setter for port + ** + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent setter for port + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPort(Integer port) + { + this.port = port; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + ** + *******************************************************************************/ + public String getPassword() + { + return password; + } + + + + /******************************************************************************* + ** Setter for password + ** + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent setter for password + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Getter for basePath + ** + *******************************************************************************/ + public String getBasePath() + { + return basePath; + } + + + + /******************************************************************************* + ** Setter for basePath + ** + *******************************************************************************/ + public void setBasePath(String basePath) + { + this.basePath = basePath; + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withBasePath(String basePath) + { + this.basePath = basePath; + return (this); + } + + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class SFTPTestConnectionTestOutput + { + private Boolean isConnectionSuccess; + private String connectionErrorMessage; + + private Boolean isListBasePathSuccess; + private String listBasePathErrorMessage; + + + + /******************************************************************************* + ** Getter for isSuccess + ** + *******************************************************************************/ + public Boolean getIsConnectionSuccess() + { + return isConnectionSuccess; + } + + + + /******************************************************************************* + ** Setter for isSuccess + ** + *******************************************************************************/ + public void setIsConnectionSuccess(Boolean isSuccess) + { + this.isConnectionSuccess = isSuccess; + } + + + + /******************************************************************************* + ** Fluent setter for isSuccess + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withIsConnectionSuccess(Boolean isSuccess) + { + this.isConnectionSuccess = isSuccess; + return (this); + } + + + + /******************************************************************************* + ** Getter for connectionErrorMessage + ** + *******************************************************************************/ + public String getConnectionErrorMessage() + { + return connectionErrorMessage; + } + + + + /******************************************************************************* + ** Setter for connectionErrorMessage + ** + *******************************************************************************/ + public void setConnectionErrorMessage(String connectionErrorMessage) + { + this.connectionErrorMessage = connectionErrorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for connectionErrorMessage + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withConnectionErrorMessage(String connectionErrorMessage) + { + this.connectionErrorMessage = connectionErrorMessage; + return (this); + } + + + + /******************************************************************************* + ** Getter for listBasePathErrorMessage + *******************************************************************************/ + public String getListBasePathErrorMessage() + { + return (this.listBasePathErrorMessage); + } + + + + /******************************************************************************* + ** Setter for listBasePathErrorMessage + *******************************************************************************/ + public void setListBasePathErrorMessage(String listBasePathErrorMessage) + { + this.listBasePathErrorMessage = listBasePathErrorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for listBasePathErrorMessage + *******************************************************************************/ + public SFTPTestConnectionTestOutput withListBasePathErrorMessage(String listBasePathErrorMessage) + { + this.listBasePathErrorMessage = listBasePathErrorMessage; + return (this); + } + + + + /******************************************************************************* + ** Getter for isListBasePathSuccess + ** + *******************************************************************************/ + public Boolean getIsListBasePathSuccess() + { + return isListBasePathSuccess; + } + + + + /******************************************************************************* + ** Setter for isListBasePathSuccess + ** + *******************************************************************************/ + public void setIsListBasePathSuccess(Boolean isListBasePathSuccess) + { + this.isListBasePathSuccess = isListBasePathSuccess; + } + + + + /******************************************************************************* + ** Fluent setter for isListBasePathSuccess + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withIsListBasePathSuccess(Boolean isListBasePathSuccess) + { + this.isListBasePathSuccess = isListBasePathSuccess; + return (this); + } + + } + +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java new file mode 100644 index 00000000..b1439cbf --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java @@ -0,0 +1,178 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for SFTPTestConnectionAction + *******************************************************************************/ +class SFTPTestConnectionActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessWithoutPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessWithPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME) + .withBasePath(BaseSFTPTest.BACKEND_FOLDER); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertTrue(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessfulConnectFailedPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME) + .withBasePath("no-such-path"); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertFalse(output.getIsListBasePathSuccess()); + assertNotNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadUsername() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername("not-" + BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadPassword() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword("not-" + BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadHostname() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName("not-" + BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadPort() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(10 * BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + +} \ No newline at end of file From dcf7218abf1038bdaf1d88698187555d41aa28ec Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 20:16:58 -0600 Subject: [PATCH 25/67] add basename field --- .../model/metadata/FilesystemTableMetaDataBuilder.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java index 4157ffb6..3c33eb39 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java @@ -81,21 +81,25 @@ public class FilesystemTableMetaDataBuilder .withPrimaryKeyField("fileName") .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("baseName", QFieldType.STRING)) .withField(new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS)) .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("contents", QFieldType.BLOB) .withIsHeavy(true) .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) - .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "File Contents"))) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s") + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName") + )) .withSection(SectionFactory.defaultT1("fileName")) - .withSection(SectionFactory.defaultT2("contents", "size")) + .withSection(SectionFactory.defaultT2("baseName", "contents", "size")) .withSection(SectionFactory.defaultT3("createDate", "modifyDate")) .withBackendDetails(tableBackendDetails .withCardinality(Cardinality.ONE) .withFileNameFieldName("fileName") + .withBaseNameFieldName("baseName") .withContentsFieldName("contents") .withSizeFieldName("size") .withCreateDateFieldName("createDate") From 2502d102d97ea9aca663a5edcb899e946a71210e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 20:17:01 -0600 Subject: [PATCH 26/67] Better version (i hope) of using ssh & sftp client objects --- .../sftp/actions/AbstractSFTPAction.java | 68 ++++++++++++++----- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index c9e3c8b5..18cb6c70 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -62,7 +62,43 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction closer = closable -> { - if(closable != null) - { - try - { - closable.close(); - } - catch(Exception e) - { - LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName())); - } - } + if(closable != null) + { + try + { + closable.close(); + } + catch(Exception e) + { + LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName())); + } + } }; - closer.accept(sshClient); - closer.accept(clientSession); closer.accept(sftpClient); + closer.accept(clientSession); } @@ -164,10 +199,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction Date: Wed, 19 Feb 2025 20:17:56 -0600 Subject: [PATCH 27/67] Update expected error message --- .../module/filesystem/sftp/actions/SFTPQueryActionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java index 9cf0234a..9708f68e 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -141,7 +141,7 @@ class SFTPQueryActionTest extends BaseSFTPTest QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); assertThatThrownBy(() -> new QueryAction().execute(queryInput)) - .hasMessageContaining("Could not find Backend Variant information for Backend"); + .hasMessageContaining("Could not find Backend Variant information in session under key 'variant-options-table' for Backend"); QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); From d25eb6ee485cd72616a523c34985d909a2bd2538 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 20 Feb 2025 11:41:29 -0600 Subject: [PATCH 28/67] Simplify file listing by replacing filters with requested paths Refactor file listing mechanisms to replace the use of complex query filters with simpler, path-based requests. Updated module-specific implementations and removed unused filtering logic. Updated tests (zombie'ing some) --- .../actions/AbstractBaseFilesystemAction.java | 45 +++++++---- .../SharedFilesystemBackendModuleUtils.java | 6 +- .../actions/AbstractFilesystemAction.java | 13 +++- .../s3/actions/AbstractS3Action.java | 5 +- .../module/filesystem/s3/utils/S3Utils.java | 68 ++++------------- .../sftp/actions/AbstractSFTPAction.java | 19 +---- .../local/FilesystemBackendModuleTest.java | 76 +++++++++---------- .../filesystem/s3/S3BackendModuleTest.java | 74 +++++++++--------- .../sftp/actions/SFTPQueryActionTest.java | 67 ---------------- 9 files changed, 143 insertions(+), 230 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 0557e68a..76a28706 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -57,8 +57,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSett import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; @@ -111,9 +111,10 @@ public abstract class AbstractBaseFilesystemAction public abstract Instant getFileModifyDate(FILE file); /******************************************************************************* - ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses. + ** List the files for a table - or optionally, just a single file name - + ** to be implemented in module-specific subclasses. *******************************************************************************/ - public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException; + public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedSingleFileName) throws QException; /******************************************************************************* ** Read the contents of a file - to be implemented in module-specific subclasses. @@ -278,11 +279,26 @@ public abstract class AbstractBaseFilesystemAction try { - QueryOutput queryOutput = new QueryOutput(queryInput); - QTableMetaData table = queryInput.getTable(); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - List files = listFiles(table, queryInput.getBackend(), queryInput.getFilter()); + + QueryOutput queryOutput = new QueryOutput(queryInput); + + String requestedPath = null; + QQueryFilter filter = queryInput.getFilter(); + if(filter != null && tableDetails.getCardinality().equals(Cardinality.ONE)) + { + if(filter.getCriteria() != null && filter.getCriteria().size() == 1) + { + QFilterCriteria criteria = filter.getCriteria().get(0); + if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) + { + requestedPath = ValueUtils.getValueAsString(criteria.getValues().get(0)); + } + } + } + + List files = listFiles(table, queryInput.getBackend(), requestedPath); switch(tableDetails.getCardinality()) { @@ -305,6 +321,7 @@ public abstract class AbstractBaseFilesystemAction } + /*************************************************************************** ** ***************************************************************************/ @@ -324,6 +341,7 @@ public abstract class AbstractBaseFilesystemAction } + /*************************************************************************** ** ***************************************************************************/ @@ -382,13 +400,12 @@ public abstract class AbstractBaseFilesystemAction // if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away // ////////////////////////////////////////////////////////////////////////////////////////////////////////// QQueryFilter filterForRecords = queryInput.getFilter(); - if(filterForRecords != null) - { - filterForRecords = filterForRecords.clone(); - - CollectionUtils.nonNullList(filterForRecords.getCriteria()) - .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria); - } + // if(filterForRecords != null) + // { + // filterForRecords = filterForRecords.clone(); + // CollectionUtils.nonNullList(filterForRecords.getCriteria()) + // .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria); + // } if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record)) { @@ -560,6 +577,7 @@ public abstract class AbstractBaseFilesystemAction } + /*************************************************************************** ** Method that subclasses can override to add post-action things (e.g., closing resources) ***************************************************************************/ @@ -571,6 +589,7 @@ public abstract class AbstractBaseFilesystemAction } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java index 026386b0..295aee06 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java @@ -130,7 +130,11 @@ public class SharedFilesystemBackendModuleUtils } else { - throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + /////////////////////////////////////////////////////////////////////////////////////////////// + // this happens in base class now, like, for query action, so, we think okay to just ignore. // + /////////////////////////////////////////////////////////////////////////////////////////////// + // throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + return (true); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 5dde1ab6..19090262 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -41,6 +41,8 @@ import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -111,7 +113,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** List the files for this table. *******************************************************************************/ @Override - public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { try { @@ -130,7 +132,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction for(String matchedFile : matchedFiles) { - if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails)) + boolean isMatch = true; + if(StringUtils.hasContent(requestedPath)) + { + QQueryFilter filter = new QQueryFilter(new QFilterCriteria(tableBackendDetails.getFileNameFieldName(), QCriteriaOperator.EQUALS, requestedPath)); + isMatch = SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails); + } + + if(isMatch) { rs.add(new File(fullPath + File.separatorChar + matchedFile)); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 4c5e7b8c..3693d32a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -33,7 +33,6 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -163,7 +162,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); @@ -175,7 +174,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, String requestedPath, AbstractFilesystemTableBackendDetails tableDetails) throws QException { ////////////////////////////////////////////////////////////////////////////////////////////////// // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // @@ -96,38 +92,20 @@ public class S3Utils prefix = prefix.substring(0, prefix.indexOf('*')); } - /////////////////////////////////////////////////////////////////////////////////////////////////////// - // for a file-per-record (ONE) table, we may need to apply the filter to listing. // - // but for MANY tables, the filtering would be done on the records after they came out of the files. // - /////////////////////////////////////////////////////////////////////////////////////////////////////// - boolean useQQueryFilter = false; - if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality())) + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // optimization, to avoid listing whole bucket, for use-case where less than a whole bucket is requested // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(requestedPath)) { - useQQueryFilter = true; - } - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if there's a filter for single file, make that file name the "prefix" that we send to s3, so we just get back that 1 file. // - // as this will be a common case. // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(filter != null && useQQueryFilter) - { - if(filter.getCriteria() != null && filter.getCriteria().size() == 1) + if(!prefix.isEmpty()) { - QFilterCriteria criteria = filter.getCriteria().get(0); - if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) - { - if(!prefix.isEmpty()) - { - /////////////////////////////////////////////////////// - // remember, a prefix starting with / finds nothing! // - /////////////////////////////////////////////////////// - prefix += "/"; - } - - prefix += criteria.getValues().get(0); - } + /////////////////////////////////////////////////////// + // remember, a prefix starting with / finds nothing! // + /////////////////////////////////////////////////////// + prefix += "/"; } + + prefix += requestedPath; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -203,27 +181,7 @@ public class S3Utils continue; } - /////////////////////////////////////////////////////////////////////////////////// - // if we're a file-per-record table, and we have a filter, compare the key to it // - /////////////////////////////////////////////////////////////////////////////////// - if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails)) - { - continue; - } - rs.add(objectSummary); - - ///////////////////////////////////////////////////////////////// - // if we have a limit, and we've hit it, break out of the loop // - ///////////////////////////////////////////////////////////////// - if(filter != null && useQQueryFilter && filter.getLimit() != null) - { - if(rs.size() >= filter.getLimit()) - { - break; - } - } - } } while(listObjectsV2Result.isTruncated()); diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 18cb6c70..1bb8a303 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -34,15 +34,13 @@ import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; @@ -267,23 +265,14 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { try { String fullPath = getFullBasePath(table, backendBase); - - // todo - move somewhere shared - // todo - should all do this? - if(filter != null) + if(StringUtils.hasContent(requestedPath)) { - for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria())) - { - if(isPathEqualsCriteria(criteria)) - { - fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar); - } - } + fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + requestedPath + File.separatorChar); } List rs = new ArrayList<>(); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index 26cb44ea..117e2cc6 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -26,9 +26,6 @@ import java.io.File; import java.io.IOException; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -41,7 +38,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -97,55 +93,59 @@ public class FilesystemBackendModuleTest ///////////////////////////////////////// // filter for a file name that's found // ///////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + files = abstractFilesystemAction.listFiles(table, backend, "BLOB-2.txt"); assertEquals(1, files.size()); assertEquals("BLOB-2.txt", files.get(0).getName()); - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + files = abstractFilesystemAction.listFiles(table, backend, "BLOB-1.txt"); assertEquals(1, files.size()); assertEquals("BLOB-1.txt", files.get(0).getName()); - /////////////////////////////////// - // filter for 2 names that exist // - /////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); - assertEquals(2, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // /////////////////////////////////// + // // filter for 2 names that exist // + // /////////////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + // assertEquals(2, files.size()); ///////////////////////////////////////////// // filter for a file name that isn't found // ///////////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + files = abstractFilesystemAction.listFiles(table, backend, "NOT-FOUND.txt"); assertEquals(0, files.size()); - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); - assertEquals(1, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + // assertEquals(1, files.size()); - //////////////////////////////////////////////////// - // 2 criteria, and'ed, and can't match, so find 0 // - //////////////////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); - assertEquals(0, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // //////////////////////////////////////////////////// + // // 2 criteria, and'ed, and can't match, so find 0 // + // //////////////////////////////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + // assertEquals(0, files.size()); - ////////////////////////////////////////////////// - // 2 criteria, or'ed, and both match, so find 2 // - ////////////////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) - .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); - assertEquals(2, files.size()); + // ////////////////////////////////////////////////// + // // 2 criteria, or'ed, and both match, so find 2 // + // ////////////////////////////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + // .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + // assertEquals(2, files.size()); - ////////////////////////////////////// - // ensure unsupported filters throw // - ////////////////////////////////////// - assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) - .rootCause() - .hasMessageContaining("Unable to query filesystem table by field"); - assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) - .rootCause() - .hasMessageContaining("Unable to query filename field using operator"); + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note that we used to try unsupported filters here, expecting them to throw - but those are // + // more-or-less now implemented in the base class's query method, so, no longer expected to throw here. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java index c18ce4bc..faec900e 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -26,9 +26,6 @@ import java.util.List; import java.util.UUID; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -38,7 +35,6 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -77,53 +73,59 @@ public class S3BackendModuleTest extends BaseS3Test ///////////////////////////////////////// // filter for a file name that's found // ///////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + files = actionBase.listFiles(table, backend, "BLOB-2.txt"); assertEquals(1, files.size()); assertThat(files.get(0).getKey()).contains("BLOB-2.txt"); - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + files = actionBase.listFiles(table, backend, "BLOB-1.txt"); assertEquals(1, files.size()); assertThat(files.get(0).getKey()).contains("BLOB-1.txt"); - /////////////////////////////////// - // filter for 2 names that exist // - /////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); - assertEquals(2, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // /////////////////////////////////// + // // filter for 2 names that exist // + // /////////////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + // assertEquals(2, files.size()); ///////////////////////////////////////////// // filter for a file name that isn't found // ///////////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + files = actionBase.listFiles(table, backend, "NOT-FOUND.txt"); assertEquals(0, files.size()); - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); - assertEquals(1, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + // assertEquals(1, files.size()); - //////////////////////////////////////////////////// - // 2 criteria, and'ed, and can't match, so find 0 // - //////////////////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); - assertEquals(0, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // //////////////////////////////////////////////////// + // // 2 criteria, and'ed, and can't match, so find 0 // + // //////////////////////////////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + // assertEquals(0, files.size()); - ////////////////////////////////////////////////// - // 2 criteria, or'ed, and both match, so find 2 // - ////////////////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) - .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); - assertEquals(2, files.size()); + // ////////////////////////////////////////////////// + // // 2 criteria, or'ed, and both match, so find 2 // + // ////////////////////////////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + // .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + // assertEquals(2, files.size()); - ////////////////////////////////////// - // ensure unsupported filters throw // - ////////////////////////////////////// - assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) - .hasMessageContaining("Unable to query filesystem table by field"); - assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) - .hasMessageContaining("Unable to query filename field using operator"); + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note that we used to try unsupported filters here, expecting them to throw - but those are // + // more-or-less now implemented in the base class's query method, so, no longer expected to throw here. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java index 9708f68e..092ea987 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -23,15 +23,11 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; import java.util.List; -import java.util.Map; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -62,69 +58,6 @@ class SFTPQueryActionTest extends BaseSFTPTest - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testQueryWithPath() throws Exception - { - String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/"; - try - { - copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt"); - copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt"); - - QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE) - .withFilter(new QQueryFilter(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - Assertions.assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows from subfolder path query"); - } - finally - { - rmrfInContainer(subfolderPath); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testQueryWithPathAndNameLike() throws Exception - { - String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/"; - try - { - copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt"); - copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt"); - copyFileToContainer("files/testfile.txt", subfolderPath + "/who.txt"); - - Map patternExpectedCountMap = Map.of( - "%.txt", 3, - "sub%", 2, - "%1%", 1, - "%", 3, - "*", 0 - ); - - for(Map.Entry entry : patternExpectedCountMap.entrySet()) - { - QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")) - .withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey()))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey()); - } - } - finally - { - rmrfInContainer(subfolderPath); - } - } - - - /******************************************************************************* ** *******************************************************************************/ From 44236f430943d598014ad49f6b745cdc7251c35d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 20 Feb 2025 11:42:23 -0600 Subject: [PATCH 29/67] change to not include createDate field for s3 (where it's not supported); changed file-name field used on the download adornment to be baseName by default, but configurable --- .../FilesystemTableMetaDataBuilder.java | 106 +++++++++++++----- 1 file changed, 80 insertions(+), 26 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java index 3c33eb39..7e61300f 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java @@ -22,12 +22,15 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; +import java.util.ArrayList; +import java.util.List; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory; import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; @@ -59,6 +62,8 @@ public class FilesystemTableMetaDataBuilder private String basePath; private String glob; + private String contentsAdornmentFileNameField = "baseName"; + /******************************************************************************* @@ -66,46 +71,64 @@ public class FilesystemTableMetaDataBuilder *******************************************************************************/ public QTableMetaData buildStandardCardinalityOneTable() { + boolean includeCreateDate = true; AbstractFilesystemTableBackendDetails tableBackendDetails = switch(backend.getBackendType()) { - case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails(); + case S3BackendModule.BACKEND_TYPE -> + { + includeCreateDate = false; + yield new S3TableBackendDetails(); + } case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails(); case SFTPBackendModule.BACKEND_TYPE -> new SFTPTableBackendDetails(); default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType()); }; + List fields = new ArrayList<>(); + + fields.add((new QFieldMetaData("fileName", QFieldType.STRING))); + fields.add((new QFieldMetaData("baseName", QFieldType.STRING))); + fields.add((new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS))); + fields.add((new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))); + fields.add((new QFieldMetaData("contents", QFieldType.BLOB) + .withIsHeavy(true) + .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s") + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, contentsAdornmentFileNameField + )))); + + QFieldSection t3Section = SectionFactory.defaultT3("modifyDate"); + + AbstractFilesystemTableBackendDetails backendDetails = tableBackendDetails + .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withBaseNameFieldName("baseName") + .withContentsFieldName("contents") + .withSizeFieldName("size") + .withModifyDateFieldName("modifyDate") + .withBasePath(basePath) + .withGlob(glob); + + if(includeCreateDate) + { + fields.add((new QFieldMetaData("createDate", QFieldType.DATE_TIME))); + backendDetails.setCreateDateFieldName("createDate"); + + ArrayList t3FieldNames = new ArrayList<>(t3Section.getFieldNames()); + t3FieldNames.add(0, "createDate"); + t3Section.setFieldNames(t3FieldNames); + } + return new QTableMetaData() .withName(name) .withIsHidden(true) .withBackendName(backend.getName()) .withPrimaryKeyField("fileName") - - .withField(new QFieldMetaData("fileName", QFieldType.STRING)) - .withField(new QFieldMetaData("baseName", QFieldType.STRING)) - .withField(new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS)) - .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) - .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) - .withField(new QFieldMetaData("contents", QFieldType.BLOB) - .withIsHeavy(true) - .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) - .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s") - .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName") - )) - + .withFields(fields) .withSection(SectionFactory.defaultT1("fileName")) .withSection(SectionFactory.defaultT2("baseName", "contents", "size")) - .withSection(SectionFactory.defaultT3("createDate", "modifyDate")) - - .withBackendDetails(tableBackendDetails - .withCardinality(Cardinality.ONE) - .withFileNameFieldName("fileName") - .withBaseNameFieldName("baseName") - .withContentsFieldName("contents") - .withSizeFieldName("size") - .withCreateDateFieldName("createDate") - .withModifyDateFieldName("modifyDate") - .withBasePath(basePath) - .withGlob(glob)); + .withSection(t3Section) + .withBackendDetails(backendDetails); } @@ -232,4 +255,35 @@ public class FilesystemTableMetaDataBuilder return (this); } + + /******************************************************************************* + ** Getter for contentsAdornmentFileNameField + *******************************************************************************/ + public String getContentsAdornmentFileNameField() + { + return (this.contentsAdornmentFileNameField); + } + + + + /******************************************************************************* + ** Setter for contentsAdornmentFileNameField + *******************************************************************************/ + public void setContentsAdornmentFileNameField(String contentsAdornmentFileNameField) + { + this.contentsAdornmentFileNameField = contentsAdornmentFileNameField; + } + + + + /******************************************************************************* + ** Fluent setter for contentsAdornmentFileNameField + *******************************************************************************/ + public FilesystemTableMetaDataBuilder withContentsAdornmentFileNameField(String contentsAdornmentFileNameField) + { + this.contentsAdornmentFileNameField = contentsAdornmentFileNameField; + return (this); + } + + } From d401cc9ae183d3cb0ea4e371897c4ccb13f78f3b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 20 Feb 2025 14:29:08 -0600 Subject: [PATCH 30/67] Implement and test DeleteAction functionality - Unified `deleteFile` API across storage modules by removing unused `QInstance` parameter. - Added implementations for S3, SFTP, and local filesystem deleteAction. --- .../actions/AbstractBaseFilesystemAction.java | 61 ++++++++++++++- .../actions/AbstractFilesystemAction.java | 2 +- .../local/actions/FilesystemDeleteAction.java | 27 +++---- .../basic/BasicETLCleanupSourceFilesStep.java | 2 +- .../importer/FilesystemImporterStep.java | 2 +- .../s3/actions/AbstractS3Action.java | 5 +- .../filesystem/s3/actions/S3DeleteAction.java | 27 +++---- .../sftp/actions/AbstractSFTPAction.java | 11 ++- .../sftp/actions/SFTPDeleteAction.java | 27 +++---- .../local/FilesystemBackendModuleTest.java | 4 +- .../actions/FilesystemDeleteActionTest.java | 29 ++++++-- .../filesystem/s3/S3BackendModuleTest.java | 6 +- .../s3/actions/S3DeleteActionTest.java | 48 +++++++++++- .../module/filesystem/sftp/BaseSFTPTest.java | 7 +- .../sftp/actions/SFTPDeleteActionTest.java | 74 ++++++++++++++++++- 15 files changed, 259 insertions(+), 73 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 76a28706..30999153 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -40,6 +40,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; @@ -57,6 +59,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSett import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; @@ -138,7 +142,7 @@ public abstract class AbstractBaseFilesystemAction ** ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit *******************************************************************************/ - public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; + public abstract void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException; /******************************************************************************* ** Move a file from a source path, to a destination path. @@ -285,7 +289,7 @@ public abstract class AbstractBaseFilesystemAction QueryOutput queryOutput = new QueryOutput(queryInput); String requestedPath = null; - QQueryFilter filter = queryInput.getFilter(); + QQueryFilter filter = queryInput.getFilter(); if(filter != null && tableDetails.getCardinality().equals(Cardinality.ONE)) { if(filter.getCriteria() != null && filter.getCriteria().size() == 1) @@ -670,4 +674,57 @@ public abstract class AbstractBaseFilesystemAction } } + + + /******************************************************************************* + ** + *******************************************************************************/ + protected DeleteOutput executeDelete(DeleteInput deleteInput) throws QException + { + try + { + preAction(deleteInput.getBackend()); + + DeleteOutput output = new DeleteOutput(); + output.setRecordsWithErrors(new ArrayList<>()); + + QTableMetaData table = deleteInput.getTable(); + QBackendMetaData backend = deleteInput.getBackend(); + + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + if(tableDetails.getCardinality().equals(Cardinality.ONE)) + { + int deletedCount = 0; + for(Serializable primaryKey : deleteInput.getPrimaryKeys()) + { + try + { + deleteFile(table, stripDuplicatedSlashes(getFullBasePath(table, backend) + "/" + primaryKey)); + deletedCount++; + } + catch(Exception e) + { + String message = ObjectUtils.tryElse(() -> ExceptionUtils.getRootException(e).getMessage(), "Message not available"); + output.addRecordWithError(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withError(new SystemErrorStatusMessage("Error deleting file: " + message))); + } + } + output.setDeletedRecordCount(deletedCount); + } + else + { + throw (new NotImplementedException("Delete is not implemented for filesystem tables with cardinality: " + tableDetails.getCardinality())); + } + + return (output); + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + finally + { + postAction(); + } + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 19090262..cb7d9255 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -230,7 +230,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit *******************************************************************************/ @Override - public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + public void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException { File file = new File(fileReference); if(!file.exists()) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java index 6d304f00..6dda158d 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java @@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; -import org.apache.commons.lang.NotImplementedException; /******************************************************************************* ** *******************************************************************************/ -public class FilesystemDeleteAction implements DeleteInterface +public class FilesystemDeleteAction extends AbstractFilesystemAction implements DeleteInterface { /******************************************************************************* @@ -40,21 +39,19 @@ public class FilesystemDeleteAction implements DeleteInterface *******************************************************************************/ public DeleteOutput execute(DeleteInput deleteInput) throws QException { - throw new NotImplementedException("Filesystem delete not implemented"); - /* - try - { - DeleteResult rs = new DeleteResult(); - QTableMetaData table = deleteRequest.getTable(); + return (executeDelete(deleteInput)); + } - // return rs; - } - catch(Exception e) - { - throw new QException("Error executing delete: " + e.getMessage(), e); - } - */ + + /******************************************************************************* + ** Specify whether this particular module's update action can & should fetch + ** records before updating them, e.g., for audits or "not-found-checks" + *******************************************************************************/ + @Override + public boolean supportsPreFetchQuery() + { + return (false); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java index 68a319ca..604d4c2b 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java @@ -94,7 +94,7 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep if(VALUE_DELETE.equals(moveOrDelete)) { LOG.info("Deleting ETL source file: " + sourceFile); - actionBase.deleteFile(QContext.getQInstance(), table, sourceFile); + actionBase.deleteFile(table, sourceFile); } else if(VALUE_MOVE.equals(moveOrDelete)) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java index f41ceb6b..bc0c3ec2 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java @@ -319,7 +319,7 @@ public class FilesystemImporterStep implements BackendStep { String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend); LOG.info("Removing source file", logPair("path", fullBasePath + "/" + sourceFileName), logPair("sourceTable", sourceTable.getName())); - sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName); + sourceActionBase.deleteFile(sourceTable, fullBasePath + "/" + sourceFileName); } else { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 3693d32a..3b8bcb09 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -31,6 +31,7 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -247,9 +248,9 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); + filesystemBackendModule.getActionBase().deleteFile(table, filesBeforeDelete.get(0).getAbsolutePath()); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(), @@ -191,7 +191,7 @@ public class FilesystemBackendModuleTest List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + filesystemBackendModule.getActionBase().deleteFile(table, PATH_THAT_WONT_EXIST); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(), diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java index e43a667e..ff113a24 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java @@ -22,11 +22,19 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; -import org.apache.commons.lang.NotImplementedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -34,14 +42,25 @@ import static org.junit.jupiter.api.Assertions.assertThrows; *******************************************************************************/ public class FilesystemDeleteActionTest extends FilesystemActionTest { - /******************************************************************************* ** *******************************************************************************/ @Test - public void test() throws QException + public void testSuccessfulDeleteMultiple() throws QException { - assertThrows(NotImplementedException.class, () -> new FilesystemDeleteAction().execute(new DeleteInput())); + int initialCount = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS)).getCount(); + + String filename1 = "A.txt"; + String filename2 = "B.txt"; + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withRecords(List.of( + new QRecord().withValue("fileName", filename1).withValue("contents", "bytes"), + new QRecord().withValue("fileName", filename2).withValue("contents", "bytes")))); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS)).getCount()); + + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withPrimaryKeys(List.of(filename1, filename2))); + assertEquals(2, deleteOutput.getDeletedRecordCount()); + assertEquals(0, deleteOutput.getRecordsWithErrors().size()); + assertEquals(initialCount, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS)).getCount()); } } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java index faec900e..b3b187de 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -147,7 +147,7 @@ public class S3BackendModuleTest extends BaseS3Test S3BackendModule s3BackendModule = new S3BackendModule(); AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); actionBase.setS3Utils(getS3Utils()); - actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); + actionBase.deleteFile(table, s3ObjectSummariesBeforeDelete.get(0).getKey()); List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), @@ -176,7 +176,7 @@ public class S3BackendModuleTest extends BaseS3Test AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); actionBase.setS3Utils(getS3Utils()); String path = "//" + s3ObjectSummariesBeforeDelete.get(0).getKey().replaceAll("/", "//"); - actionBase.deleteFile(qInstance, table, "//" + path); + actionBase.deleteFile(table, "//" + path); List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), @@ -203,7 +203,7 @@ public class S3BackendModuleTest extends BaseS3Test S3BackendModule s3BackendModule = new S3BackendModule(); AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); actionBase.setS3Utils(getS3Utils()); - actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + actionBase.deleteFile(table, PATH_THAT_WONT_EXIST); List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(), diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java index 6b1ba2fa..ce01677f 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java @@ -22,12 +22,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; +import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; -import org.apache.commons.lang.NotImplementedException; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -42,7 +49,42 @@ public class S3DeleteActionTest extends BaseS3Test @Test public void test() throws QException { - assertThrows(NotImplementedException.class, () -> new S3DeleteAction().execute(new DeleteInput())); + QInstance qInstance = TestUtils.defineInstance(); + + int initialCount = count(TestUtils.TABLE_NAME_BLOB_S3); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob."))); + + S3InsertAction insertAction = new S3InsertAction(); + insertAction.setS3Utils(getS3Utils()); + insertAction.execute(insertInput); + + assertEquals(initialCount + 1, count(TestUtils.TABLE_NAME_BLOB_S3)); + + S3DeleteAction deleteAction = new S3DeleteAction(); + deleteAction.setS3Utils(getS3Utils()); + DeleteOutput deleteOutput = deleteAction.execute(new DeleteInput(TestUtils.TABLE_NAME_BLOB_S3).withPrimaryKeys(List.of("file2.txt"))); + assertEquals(1, deleteOutput.getDeletedRecordCount()); + assertEquals(0, deleteOutput.getRecordsWithErrors().size()); + + assertEquals(initialCount, count(TestUtils.TABLE_NAME_BLOB_S3)); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + private Integer count(String tableName) throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(tableName); + S3CountAction s3CountAction = new S3CountAction(); + s3CountAction.setS3Utils(getS3Utils()); + CountOutput countOutput = s3CountAction.execute(countInput); + return countOutput.getCount(); } } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java index 42c968be..6c71ea42 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java @@ -43,8 +43,8 @@ public class BaseSFTPTest extends BaseTest public static final String TABLE_FOLDER = "files"; public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER; - private static GenericContainer sftpContainer; - private static Integer currentPort; + protected static GenericContainer sftpContainer; + private static Integer currentPort; @@ -71,6 +71,7 @@ public class BaseSFTPTest extends BaseTest } + /*************************************************************************** ** ***************************************************************************/ @@ -80,6 +81,7 @@ public class BaseSFTPTest extends BaseTest } + /*************************************************************************** ** ***************************************************************************/ @@ -89,6 +91,7 @@ public class BaseSFTPTest extends BaseTest } + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java index 3a992fbf..15d74d3b 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java @@ -22,12 +22,23 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; -import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -35,14 +46,69 @@ import static org.junit.jupiter.api.Assertions.assertThrows; *******************************************************************************/ public class SFTPDeleteActionTest extends BaseSFTPTest { + private String filesBasename = "delete-test-"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() throws Exception + { + rmrfInContainer(REMOTE_DIR + "/" + filesBasename + "*"); + } + + /******************************************************************************* ** *******************************************************************************/ @Test - public void test() throws QException + public void testSuccessfulDeleteMultiple() throws QException { - assertThrows(NotImplementedException.class, () -> new SFTPDeleteAction().execute(new DeleteInput())); + int initialCount = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount(); + + String filename1 = filesBasename + "A.txt"; + String filename2 = filesBasename + "B.txt"; + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SFTP_FILE).withRecords(List.of( + new QRecord().withValue("fileName", filename1).withValue("contents", "bytes"), + new QRecord().withValue("fileName", filename2).withValue("contents", "bytes")))); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_SFTP_FILE).withPrimaryKeys(List.of(filename1, filename2))); + assertEquals(2, deleteOutput.getDeletedRecordCount()); + assertEquals(0, deleteOutput.getRecordsWithErrors().size()); + assertEquals(initialCount, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFailedDelete() throws Exception + { + int initialCount = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount(); + + String filename1 = filesBasename + "C.txt"; + String filename2 = filesBasename + "D.txt"; + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SFTP_FILE).withRecords(List.of( + new QRecord().withValue("fileName", filename1).withValue("contents", "bytes"), + new QRecord().withValue("fileName", filename2).withValue("contents", "bytes")))); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + + sftpContainer.execInContainer("chmod", "000", REMOTE_DIR); + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_SFTP_FILE).withPrimaryKeys(List.of(filename1, filename2))); + sftpContainer.execInContainer("chmod", "777", REMOTE_DIR); + + assertEquals(0, deleteOutput.getDeletedRecordCount()); + assertEquals(2, deleteOutput.getRecordsWithErrors().size()); + assertThat(deleteOutput.getRecordsWithErrors().get(0).getErrorsAsString()).contains("Error deleting file: Permission denied"); + assertThat(deleteOutput.getRecordsWithErrors().get(1).getErrorsAsString()).contains("Error deleting file: Permission denied"); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); } } \ No newline at end of file From 05bb0ef363d4778d9652558f5844531da652076a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 20 Feb 2025 15:35:52 -0600 Subject: [PATCH 31/67] Fix withoutCapabilities(Set) - was calling with, not without :( --- .../qqq/backend/core/model/metadata/tables/QTableMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 1df589d1..0d0d6152 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -1056,7 +1056,7 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData { for(Capability disabledCapability : disabledCapabilities) { - withCapability(disabledCapability); + withoutCapability(disabledCapability); } return (this); } From 693dfb2d5b968285f19e39f9d487ec8efea65bfb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 15:02:29 -0600 Subject: [PATCH 32/67] Update getExistingRecordQueryFilter to convert sourceKeyList to be in the destination foreign key field's type --- .../AbstractTableSyncTransformStep.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index b14847f7..c8f6b82e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -173,8 +173,21 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt *******************************************************************************/ protected QQueryFilter getExistingRecordQueryFilter(RunBackendStepInput runBackendStepInput, List sourceKeyList) { - String destinationTableForeignKeyField = getSyncProcessConfig().destinationTableForeignKey; - return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList)); + String destinationTableForeignKeyFieldName = getSyncProcessConfig().destinationTableForeignKey; + String destinationTableName = getSyncProcessConfig().destinationTable; + QFieldMetaData destinationForeignKeyField = QContext.getQInstance().getTable(destinationTableName).getField(destinationTableForeignKeyFieldName); + + List sourceKeysInDestinationKeyTypeList = null; + if(sourceKeyList != null) + { + sourceKeysInDestinationKeyTypeList = new ArrayList<>(); + for(Serializable sourceKey : sourceKeyList) + { + sourceKeysInDestinationKeyTypeList.add(ValueUtils.getValueAsFieldType(destinationForeignKeyField.getType(), sourceKey)); + } + } + + return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyFieldName, QCriteriaOperator.IN, sourceKeysInDestinationKeyTypeList)); } From df530b70b85021e78c326badde29e2af91ec9bca Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 15:04:02 -0600 Subject: [PATCH 33/67] Add static wrapper --- .../core/actions/tables/CountAction.java | 17 +++++++++++++++++ .../core/actions/tables/CountActionTest.java | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java index 5a25c396..497d6f34 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java @@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; @@ -82,6 +83,22 @@ public class CountAction + /******************************************************************************* + ** shorthand way to call for the most common use-case, when you just want the + ** count to be returned, and you just want to pass in a table name and filter. + *******************************************************************************/ + public static Integer execute(String tableName, QQueryFilter filter) throws QException + { + CountAction countAction = new CountAction(); + CountInput countInput = new CountInput(); + countInput.setTableName(tableName); + countInput.setFilter(filter); + CountOutput countOutput = countAction.execute(countInput); + return (countOutput.getCount()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java index ba1c7c9b..5a93f970 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java @@ -23,10 +23,14 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -50,4 +54,18 @@ class CountActionTest extends BaseTest CountOutput result = new CountAction().execute(request); assertNotNull(result); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStaticWrapper() throws QException + { + TestUtils.insertDefaultShapes(QContext.getQInstance()); + assertEquals(3, CountAction.execute(TestUtils.TABLE_NAME_SHAPE, null)); + assertEquals(3, CountAction.execute(TestUtils.TABLE_NAME_SHAPE, new QQueryFilter())); + } + } From 0395e0d02cfdd77c75aec5def34f094cb0696dbd Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 15:04:23 -0600 Subject: [PATCH 34/67] Add warning if input primaryKey is a filter (because that's probably not what you wanted!) --- .../kingsrook/qqq/backend/core/actions/tables/GetAction.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index 24c6e983..c98e31f3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -238,6 +238,11 @@ public class GetAction *******************************************************************************/ public static QRecord execute(String tableName, Serializable primaryKey) throws QException { + if(primaryKey instanceof QQueryFilter) + { + LOG.warn("Unexpected use of QQueryFilter instead of primary key in GetAction call"); + } + GetAction getAction = new GetAction(); GetInput getInput = new GetInput(tableName).withPrimaryKey(primaryKey); return getAction.executeForRecord(getInput); From f4f2f3c80e67f208f7253ed9bc41e54d9cf56877 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 15:05:11 -0600 Subject: [PATCH 35/67] Refactor a findProducers method out of processAllMetaDataProducersInPackage, for more flexibility (e.g., in QBitProducers) --- .../metadata/MetaDataProducerHelper.java | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index d4600787..487c6dda 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -106,14 +106,10 @@ public class MetaDataProducerHelper } - - /******************************************************************************* - ** Recursively find all classes in the given package, that implement MetaDataProducerInterface - ** run them, and add their output to the given qInstance. + /*************************************************************************** ** - ** Note - they'll be sorted by the sortOrder they provide. - *******************************************************************************/ - public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException + ***************************************************************************/ + public static List> findProducers(String packageName) throws QException { List> classesInPackage; try @@ -196,6 +192,20 @@ public class MetaDataProducerHelper } })); + return (producers); + } + + + /******************************************************************************* + ** Recursively find all classes in the given package, that implement MetaDataProducerInterface + ** run them, and add their output to the given qInstance. + ** + ** Note - they'll be sorted by the sortOrder they provide. + *******************************************************************************/ + public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException + { + List> producers = findProducers(packageName); + /////////////////////////////////////////////////////////////////////////// // execute each one (if enabled), adding their meta data to the instance // /////////////////////////////////////////////////////////////////////////// From 001860fc91e743f0ca221849501695b24bb2e13a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:24:30 -0600 Subject: [PATCH 36/67] Initial checkin of QBits, mostly. --- .../metadata/MetaDataProducerMultiOutput.java | 49 +++- .../core/model/metadata/QInstance.java | 67 +++++ .../metadata/processes/QProcessMetaData.java | 46 +++- .../qbits/ProvidedOrSuppliedTableConfig.java | 122 +++++++++ .../qbits/QBitComponentMetaDataProducer.java | 71 ++++++ .../core/model/metadata/qbits/QBitConfig.java | 110 ++++++++ .../qbits/QBitConfigValidationException.java | 44 ++++ .../model/metadata/qbits/QBitMetaData.java | 237 ++++++++++++++++++ .../model/metadata/qbits/QBitProducer.java | 117 +++++++++ .../model/metadata/qbits/SourceQBitAware.java | 77 ++++++ .../model/metadata/tables/QTableMetaData.java | 39 ++- .../metadata/qbits/QBitProducerTest.java | 151 +++++++++++ .../qbits/testqbit/TestQBitConfig.java | 181 +++++++++++++ .../qbits/testqbit/TestQBitProducer.java | 92 +++++++ .../metadata/OtherTableMetaDataProducer.java | 69 +++++ .../metadata/SomeTableMetaDataProducer.java | 68 +++++ 16 files changed, 1534 insertions(+), 6 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java index d6797a2a..32c446a1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -31,10 +32,12 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; ** Output object for a MetaDataProducer, which contains multiple meta-data ** objects. *******************************************************************************/ -public class MetaDataProducerMultiOutput implements MetaDataProducerOutput +public class MetaDataProducerMultiOutput implements MetaDataProducerOutput, SourceQBitAware { private List contents; + private String sourceQBitName; + /******************************************************************************* @@ -98,4 +101,48 @@ public class MetaDataProducerMultiOutput implements MetaDataProducerOutput return (rs); } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getSourceQBitName() + { + return (this.sourceQBitName); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void setSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + + ///////////////////////////////////////////// + // propagate the name down to the children // + ///////////////////////////////////////////// + for(MetaDataProducerOutput content : contents) + { + if(content instanceof SourceQBitAware aware) + { + aware.setSourceQBitName(sourceQBitName); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public MetaDataProducerMultiOutput withSourceQBitName(String sourceQBitName) + { + setSourceQBitName(sourceQBitName); + return this; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index a61971c8..def809b9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -56,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRule import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; @@ -89,6 +90,7 @@ public class QInstance //////////////////////////////////////////////////////////////////////////////////////////// // Important to use LinkedHashmap here, to preserve the order in which entries are added. // //////////////////////////////////////////////////////////////////////////////////////////// + private Map qBits = new LinkedHashMap<>(); private Map tables = new LinkedHashMap<>(); private Map joins = new LinkedHashMap<>(); private Map possibleValueSources = new LinkedHashMap<>(); @@ -1489,6 +1491,7 @@ public class QInstance } + /******************************************************************************* ** Getter for metaDataFilter *******************************************************************************/ @@ -1519,4 +1522,68 @@ public class QInstance } + + /******************************************************************************* + ** + *******************************************************************************/ + public void addQBit(QBitMetaData qBitMetaData) + { + List missingParts = new ArrayList<>(); + if(!StringUtils.hasContent(qBitMetaData.getGroupId())) + { + missingParts.add("groupId"); + } + if(!StringUtils.hasContent(qBitMetaData.getArtifactId())) + { + missingParts.add("artifactId"); + } + if(!StringUtils.hasContent(qBitMetaData.getVersion())) + { + missingParts.add("version"); + + } + if(!missingParts.isEmpty()) + { + throw (new IllegalArgumentException("Attempted to add a qBit without a " + StringUtils.joinWithCommasAndAnd(missingParts))); + } + + String name = qBitMetaData.getName(); + if(this.qBits.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second qBit with name (formed from 'groupId:artifactId:version[:namespace]'): " + name)); + } + this.qBits.put(name, qBitMetaData); + } + + + + /******************************************************************************* + ** Getter for qBits + *******************************************************************************/ + public Map getQBits() + { + return (this.qBits); + } + + + + /******************************************************************************* + ** Setter for qBits + *******************************************************************************/ + public void setQBits(Map qBits) + { + this.qBits = qBits; + } + + + + /******************************************************************************* + ** Fluent setter for qBits + *******************************************************************************/ + public QInstance withQBits(Map qBits) + { + this.qBits = qBits; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 582b36e2..21112267 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -46,11 +47,14 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; ** Meta-Data to define a process in a QQQ instance. ** *******************************************************************************/ -public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface +public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface, SourceQBitAware { - private String name; - private String label; - private String tableName; + private String name; + private String label; + private String tableName; + + private String sourceQBitName; + private boolean isHidden = false; private BasepullConfiguration basepullConfiguration; private QPermissionRules permissionRules; @@ -870,6 +874,7 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi } + /******************************************************************************* ** Getter for processTracerCodeReference *******************************************************************************/ @@ -900,4 +905,37 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi } + /******************************************************************************* + ** Getter for sourceQBitName + *******************************************************************************/ + @Override + public String getSourceQBitName() + { + return (this.sourceQBitName); + } + + + + /******************************************************************************* + ** Setter for sourceQBitName + *******************************************************************************/ + @Override + public void setSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + } + + + + /******************************************************************************* + ** Fluent setter for sourceQBitName + *******************************************************************************/ + @Override + public QProcessMetaData withSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java new file mode 100644 index 00000000..3383a4c6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java @@ -0,0 +1,122 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +/*************************************************************************** + ** Common (maybe)? qbit config pattern, where the qbit may be able to provide + ** a particular table, or, the application may supply it itself. + ** + ** If the qbit provides it, then we need to be told (by the application) + ** what backendName to use for the table. + ** + ** Else if the application supplies it, it needs to tell the qBit what the + ** tableName is. + ***************************************************************************/ +public class ProvidedOrSuppliedTableConfig +{ + private boolean doProvideTable; + private String backendName; + private String tableName; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public ProvidedOrSuppliedTableConfig(boolean doProvideTable, String backendName, String tableName) + { + this.doProvideTable = doProvideTable; + this.backendName = backendName; + this.tableName = tableName; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProvidedOrSuppliedTableConfig provideTableUsingBackendNamed(String backendName) + { + return (new ProvidedOrSuppliedTableConfig(true, backendName, null)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProvidedOrSuppliedTableConfig useSuppliedTaleNamed(String tableName) + { + return (new ProvidedOrSuppliedTableConfig(false, null, tableName)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public String getEffectiveTableName(String tableNameIfProviding) + { + if (getDoProvideTable()) + { + return tableNameIfProviding; + } + else + { + return getTableName(); + } + } + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Getter for doProvideTable + ** + *******************************************************************************/ + public boolean getDoProvideTable() + { + return doProvideTable; + } + + + + /******************************************************************************* + ** Getter for backendName + ** + *******************************************************************************/ + public String getBackendName() + { + return backendName; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java new file mode 100644 index 00000000..346633f8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java @@ -0,0 +1,71 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput; + + +/******************************************************************************* + ** extension of MetaDataProducerInterface, designed for producing meta data + ** within a (java-defined, at this time) QBit. + ** + ** Specifically exists to accept the QBitConfig as a type parameter and a value, + ** easily accessed in the producer's methods as getQBitConfig() + *******************************************************************************/ +public abstract class QBitComponentMetaDataProducer implements MetaDataProducerInterface +{ + private C qBitConfig = null; + + + + /******************************************************************************* + ** Getter for qBitConfig + *******************************************************************************/ + public C getQBitConfig() + { + return (this.qBitConfig); + } + + + + /******************************************************************************* + ** Setter for qBitConfig + *******************************************************************************/ + public void setQBitConfig(C qBitConfig) + { + this.qBitConfig = qBitConfig; + } + + + + /******************************************************************************* + ** Fluent setter for qBitConfig + *******************************************************************************/ + public QBitComponentMetaDataProducer withQBitConfig(C qBitConfig) + { + this.qBitConfig = qBitConfig; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java new file mode 100644 index 00000000..bb4ea5c1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Interface for configuration settings used both in the production of meta-data + ** for a QBit, but also at runtime, e.g., to be aware of exactly how the qbit + ** has been incorporated into an application. + ** + ** For example: + ** - should the QBit define certain tables, or will they be supplied by the application? + ** - what other meta-data names should the qbit reference (backends, schedulers) + ** - what meta-data-customizer(s) should be used? + ** + ** When implementing a QBit, you'll implement this interface - adding whatever + ** (if any) properties you need, and if you have any rules, then overriding + ** the validate method (ideally the one that takes the List-of-String errors) + ** + ** When using a QBit, you'll create an instance of the QBit's config object, + ** and pass it through to the QBit producer. + *******************************************************************************/ +public interface QBitConfig extends Serializable +{ + QLogger LOG = QLogger.getLogger(QBitConfig.class); + + + /*************************************************************************** + ** + ***************************************************************************/ + default void validate(QInstance qInstance) throws QBitConfigValidationException + { + List errors = new ArrayList<>(); + + try + { + validate(qInstance, errors); + } + catch(Exception e) + { + LOG.warn("Error validating QBitConfig: " + this.getClass().getName(), e); + } + + if(!errors.isEmpty()) + { + throw (new QBitConfigValidationException(this, errors)); + } + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default void validate(QInstance qInstance, List errors) + { + ///////////////////////////////////// + // nothing to validate by default! // + ///////////////////////////////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default boolean assertCondition(boolean condition, String message, List errors) + { + if(!condition) + { + errors.add(message); + } + return (condition); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default MetaDataCustomizerInterface getTableMetaDataCustomizer() + { + return (null); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java new file mode 100644 index 00000000..058442c5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** thrown by QBitConfig.validate() if there's an issue. + *******************************************************************************/ +public class QBitConfigValidationException extends QException +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public QBitConfigValidationException(QBitConfig qBitConfig, List errors) + { + super("Validation failed for QBitConfig: " + qBitConfig.getClass().getName() + ":\n" + StringUtils.join("\n", errors)); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java new file mode 100644 index 00000000..feacb969 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java @@ -0,0 +1,237 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Meta-data to define an active QBit in a QQQ Instance. + ** + ** The unique "name" for the QBit is composed of its groupId and artifactId + ** (maven style). There is also a version - but it is not part of the unique + ** name. But - there is also a namespace attribute, which IS part of the + ** unique name. This will (eventually?) allow us to have multiple instances + ** of the same qbit in a qInstance at the same time (e.g., 2 versions of some + ** table, which should be namespace-prefixed); + ** + ** QBitMetaData also retains the QBitConfig that was used to produce the QBit. + ** + ** Some meta-data objects are aware of the fact that they may have come from a + ** QBit - see SourceQBitAware interface. These objects can get their source + ** QBitMetaData (this object) and its config,via that interface. + *******************************************************************************/ +public class QBitMetaData implements TopLevelMetaDataInterface +{ + private String groupId; + private String artifactId; + private String version; + private String namespace; + + private QBitConfig config; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getName() + { + String name = groupId + ":" + artifactId; + if(StringUtils.hasContent(namespace)) + { + name += ":" + namespace; + } + return name; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void addSelfToInstance(QInstance qInstance) + { + qInstance.addQBit(this); + } + + + + /******************************************************************************* + ** Getter for config + *******************************************************************************/ + public QBitConfig getConfig() + { + return (this.config); + } + + + + /******************************************************************************* + ** Setter for config + *******************************************************************************/ + public void setConfig(QBitConfig config) + { + this.config = config; + } + + + + /******************************************************************************* + ** Fluent setter for config + *******************************************************************************/ + public QBitMetaData withConfig(QBitConfig config) + { + this.config = config; + return (this); + } + + + + /******************************************************************************* + ** Getter for groupId + *******************************************************************************/ + public String getGroupId() + { + return (this.groupId); + } + + + + /******************************************************************************* + ** Setter for groupId + *******************************************************************************/ + public void setGroupId(String groupId) + { + this.groupId = groupId; + } + + + + /******************************************************************************* + ** Fluent setter for groupId + *******************************************************************************/ + public QBitMetaData withGroupId(String groupId) + { + this.groupId = groupId; + return (this); + } + + + + /******************************************************************************* + ** Getter for artifactId + *******************************************************************************/ + public String getArtifactId() + { + return (this.artifactId); + } + + + + /******************************************************************************* + ** Setter for artifactId + *******************************************************************************/ + public void setArtifactId(String artifactId) + { + this.artifactId = artifactId; + } + + + + /******************************************************************************* + ** Fluent setter for artifactId + *******************************************************************************/ + public QBitMetaData withArtifactId(String artifactId) + { + this.artifactId = artifactId; + return (this); + } + + + + /******************************************************************************* + ** Getter for version + *******************************************************************************/ + public String getVersion() + { + return (this.version); + } + + + + /******************************************************************************* + ** Setter for version + *******************************************************************************/ + public void setVersion(String version) + { + this.version = version; + } + + + + /******************************************************************************* + ** Fluent setter for version + *******************************************************************************/ + public QBitMetaData withVersion(String version) + { + this.version = version; + return (this); + } + + + + /******************************************************************************* + ** Getter for namespace + *******************************************************************************/ + public String getNamespace() + { + return (this.namespace); + } + + + + /******************************************************************************* + ** Setter for namespace + *******************************************************************************/ + public void setNamespace(String namespace) + { + this.namespace = namespace; + } + + + + /******************************************************************************* + ** Fluent setter for namespace + *******************************************************************************/ + public QBitMetaData withNamespace(String namespace) + { + this.namespace = namespace; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java new file mode 100644 index 00000000..7fef2174 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** interface for how a QBit's meta-data gets produced and added to a QInstance. + ** + ** When implementing a QBit, you'll implement this interface: + ** - adding a QBitConfig subclass as a property + ** - overriding the produce(qInstance, namespace) method - where you'll: + ** -- create and add your QBitMetaData + ** -- call MetaDataProducerHelper.findProducers + ** -- hand off to finishProducing() in this interface + ** + ** When using a QBit, you'll create an instance of the QBit's config object, + ** pass it in to the producer, then call produce, ala: + ** + ** new SomeQBitProducer() + ** .withQBitConfig(someQBitConfig) + ** .produce(qInstance); + ** + *******************************************************************************/ +public interface QBitProducer +{ + QLogger LOG = QLogger.getLogger(QBitProducer.class); + + + /*************************************************************************** + ** + ***************************************************************************/ + default void produce(QInstance qInstance) throws QException + { + produce(qInstance, null); + } + + /*************************************************************************** + ** + ***************************************************************************/ + void produce(QInstance qInstance, String namespace) throws QException; + + + /*************************************************************************** + * + ***************************************************************************/ + default void finishProducing(QInstance qInstance, QBitMetaData qBitMetaData, C qBitConfig, List> producers) throws QException + { + qBitConfig.validate(qInstance); + + /////////////////////////////// + // todo - move to base class // + /////////////////////////////// + for(MetaDataProducerInterface producer : producers) + { + if(producer instanceof QBitComponentMetaDataProducer) + { + QBitComponentMetaDataProducer qBitComponentMetaDataProducer = (QBitComponentMetaDataProducer) producer; + qBitComponentMetaDataProducer.setQBitConfig(qBitConfig); + } + + if(!producer.isEnabled()) + { + LOG.debug("Not using producer which is not enabled", logPair("producer", producer.getClass().getSimpleName())); + continue; + } + + MetaDataProducerOutput output = producer.produce(qInstance); + + ///////////////////////////////////////// + // apply table customizer, if provided // + ///////////////////////////////////////// + if(qBitConfig.getTableMetaDataCustomizer() != null && output instanceof QTableMetaData table) + { + output = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table); + } + + ///////////////////////////////////////////////// + // set source qbit, if output is aware of such // + ///////////////////////////////////////////////// + if(output instanceof SourceQBitAware sourceQBitAware) + { + sourceQBitAware.setSourceQBitName(qBitMetaData.getName()); + } + + output.addSelfToInstance(qInstance); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java new file mode 100644 index 00000000..c4d4be7f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import com.kingsrook.qqq.backend.core.context.QContext; + + +/******************************************************************************* + ** interface for meta data objects that may have come from a qbit, and where we + ** might want to get data about that qbit (e.g., config or meta-data). + *******************************************************************************/ +public interface SourceQBitAware +{ + /******************************************************************************* + ** Getter for sourceQBitName + *******************************************************************************/ + String getSourceQBitName(); + + + /******************************************************************************* + ** Setter for sourceQBitName + *******************************************************************************/ + void setSourceQBitName(String sourceQBitName); + + + /******************************************************************************* + ** Fluent setter for sourceQBitName + *******************************************************************************/ + Object withSourceQBitName(String sourceQBitName); + + + /*************************************************************************** + ** + ***************************************************************************/ + default QBitMetaData getSourceQBit() + { + String qbitName = getSourceQBitName(); + return (QContext.getQInstance().getQBits().get(qbitName)); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default QBitConfig getSourceQBitConfig() + { + QBitMetaData sourceQBit = getSourceQBit(); + if(sourceQBit == null) + { + return null; + } + else + { + return sourceQBit.getConfig(); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 0d0d6152..b2318de5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -50,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; @@ -62,7 +63,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; ** Meta-Data to define a table in a QQQ instance. ** *******************************************************************************/ -public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface +public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface, SourceQBitAware { private static final QLogger LOG = QLogger.getLogger(QTableMetaData.class); @@ -73,6 +74,8 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData private String primaryKeyField; private boolean isHidden = false; + private String sourceQBitName; + private Map fields; private List uniqueKeys; private List associations; @@ -1554,4 +1557,38 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot); } + + /******************************************************************************* + ** Getter for sourceQBitName + *******************************************************************************/ + @Override + public String getSourceQBitName() + { + return (this.sourceQBitName); + } + + + + /******************************************************************************* + ** Setter for sourceQBitName + *******************************************************************************/ + @Override + public void setSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + } + + + + /******************************************************************************* + ** Fluent setter for sourceQBitName + *******************************************************************************/ + @Override + public QTableMetaData withSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java new file mode 100644 index 00000000..de56fbfc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java @@ -0,0 +1,151 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.util.LinkedHashMap; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata.OtherTableMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata.SomeTableMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for QBitProducer + *******************************************************************************/ +class QBitProducerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + TestQBitConfig config = new TestQBitConfig() + .withOtherTableConfig(ProvidedOrSuppliedTableConfig.provideTableUsingBackendNamed(TestUtils.MEMORY_BACKEND_NAME)) + .withIsSomeTableEnabled(true) + .withSomeSetting("yes") + .withTableMetaDataCustomizer((i, table) -> + { + if(table.getBackendName() == null) + { + table.setBackendName(TestUtils.DEFAULT_BACKEND_NAME); + } + + table.addField(new QFieldMetaData("custom", QFieldType.STRING)); + + return (table); + }); + + QInstance qInstance = QContext.getQInstance(); + new TestQBitProducer().withTestQBitConfig(config).produce(qInstance); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // OtherTable should have been provided by the qbit, with the backend name we told it above (MEMORY) // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + QTableMetaData otherTable = qInstance.getTable(OtherTableMetaDataProducer.NAME); + assertNotNull(otherTable); + assertEquals(TestUtils.MEMORY_BACKEND_NAME, otherTable.getBackendName()); + assertNotNull(otherTable.getField("custom")); + + QBitMetaData sourceQBit = otherTable.getSourceQBit(); + assertEquals("testQBit", sourceQBit.getArtifactId()); + + //////////////////////////////////////////////////////////////////////////////// + // SomeTable should have been provided, w/ backend name set by the customizer // + //////////////////////////////////////////////////////////////////////////////// + QTableMetaData someTable = qInstance.getTable(SomeTableMetaDataProducer.NAME); + assertNotNull(someTable); + assertEquals(TestUtils.DEFAULT_BACKEND_NAME, someTable.getBackendName()); + assertNotNull(otherTable.getField("custom")); + + TestQBitConfig qBitConfig = (TestQBitConfig) someTable.getSourceQBitConfig(); + assertEquals("yes", qBitConfig.getSomeSetting()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDisableThings() throws QException + { + TestQBitConfig config = new TestQBitConfig() + .withOtherTableConfig(ProvidedOrSuppliedTableConfig.useSuppliedTaleNamed(TestUtils.TABLE_NAME_PERSON_MEMORY)) + .withIsSomeTableEnabled(false); + + QInstance qInstance = QContext.getQInstance(); + new TestQBitProducer().withTestQBitConfig(config).produce(qInstance); + + ////////////////////////////////////// + // neither table should be produced // + ////////////////////////////////////// + QTableMetaData otherTable = qInstance.getTable(OtherTableMetaDataProducer.NAME); + assertNull(otherTable); + + QTableMetaData someTable = qInstance.getTable(SomeTableMetaDataProducer.NAME); + assertNull(someTable); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidationErrors() throws QException + { + QInstance qInstance = QContext.getQInstance(); + TestQBitConfig config = new TestQBitConfig(); + + assertThatThrownBy(() -> new TestQBitProducer().withTestQBitConfig(config).produce(qInstance)) + .isInstanceOf(QBitConfigValidationException.class) + .hasMessageContaining("otherTableConfig must be set") + .hasMessageContaining("isSomeTableEnabled must be set"); + qInstance.setQBits(new LinkedHashMap<>()); + + config.setIsSomeTableEnabled(true); + assertThatThrownBy(() -> new TestQBitProducer().withTestQBitConfig(config).produce(qInstance)) + .isInstanceOf(QBitConfigValidationException.class) + .hasMessageContaining("otherTableConfig must be set"); + qInstance.setQBits(new LinkedHashMap<>()); + + config.setOtherTableConfig(ProvidedOrSuppliedTableConfig.useSuppliedTaleNamed(TestUtils.TABLE_NAME_PERSON_MEMORY)); + new TestQBitProducer().withTestQBitConfig(config).produce(qInstance); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java new file mode 100644 index 00000000..05ac39a9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java @@ -0,0 +1,181 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.ProvidedOrSuppliedTableConfig; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestQBitConfig implements QBitConfig +{ + private MetaDataCustomizerInterface tableMetaDataCustomizer; + + private Boolean isSomeTableEnabled; + private ProvidedOrSuppliedTableConfig otherTableConfig; + + private String someSetting; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void validate(QInstance qInstance, List errors) + { + assertCondition(otherTableConfig != null, "otherTableConfig must be set", errors); + assertCondition(isSomeTableEnabled != null, "isSomeTableEnabled must be set", errors); + } + + + + /******************************************************************************* + ** Getter for otherTableConfig + *******************************************************************************/ + public ProvidedOrSuppliedTableConfig getOtherTableConfig() + { + return (this.otherTableConfig); + } + + + + /******************************************************************************* + ** Setter for otherTableConfig + *******************************************************************************/ + public void setOtherTableConfig(ProvidedOrSuppliedTableConfig otherTableConfig) + { + this.otherTableConfig = otherTableConfig; + } + + + + /******************************************************************************* + ** Fluent setter for otherTableConfig + *******************************************************************************/ + public TestQBitConfig withOtherTableConfig(ProvidedOrSuppliedTableConfig otherTableConfig) + { + this.otherTableConfig = otherTableConfig; + return (this); + } + + + + /******************************************************************************* + ** Getter for isSomeTableEnabled + *******************************************************************************/ + public Boolean getIsSomeTableEnabled() + { + return (this.isSomeTableEnabled); + } + + + + /******************************************************************************* + ** Setter for isSomeTableEnabled + *******************************************************************************/ + public void setIsSomeTableEnabled(Boolean isSomeTableEnabled) + { + this.isSomeTableEnabled = isSomeTableEnabled; + } + + + + /******************************************************************************* + ** Fluent setter for isSomeTableEnabled + *******************************************************************************/ + public TestQBitConfig withIsSomeTableEnabled(Boolean isSomeTableEnabled) + { + this.isSomeTableEnabled = isSomeTableEnabled; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableMetaDataCustomizer + *******************************************************************************/ + public MetaDataCustomizerInterface getTableMetaDataCustomizer() + { + return (this.tableMetaDataCustomizer); + } + + + + /******************************************************************************* + ** Setter for tableMetaDataCustomizer + *******************************************************************************/ + public void setTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + this.tableMetaDataCustomizer = tableMetaDataCustomizer; + } + + + + /******************************************************************************* + ** Fluent setter for tableMetaDataCustomizer + *******************************************************************************/ + public TestQBitConfig withTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + this.tableMetaDataCustomizer = tableMetaDataCustomizer; + return (this); + } + + + + /******************************************************************************* + ** Getter for someSetting + *******************************************************************************/ + public String getSomeSetting() + { + return (this.someSetting); + } + + + + /******************************************************************************* + ** Setter for someSetting + *******************************************************************************/ + public void setSomeSetting(String someSetting) + { + this.someSetting = someSetting; + } + + + + /******************************************************************************* + ** Fluent setter for someSetting + *******************************************************************************/ + public TestQBitConfig withSomeSetting(String someSetting) + { + this.someSetting = someSetting; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java new file mode 100644 index 00000000..14c1d5d1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestQBitProducer implements QBitProducer +{ + private TestQBitConfig testQBitConfig; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void produce(QInstance qInstance, String namespace) throws QException + { + QBitMetaData qBitMetaData = new QBitMetaData() + .withGroupId("test.com.kingsrook.qbits") + .withArtifactId("testQBit") + .withVersion("0.1.0") + .withNamespace(namespace) + .withConfig(testQBitConfig); + qInstance.addQBit(qBitMetaData); + + List> producers = MetaDataProducerHelper.findProducers(getClass().getPackageName() + ".metadata"); + finishProducing(qInstance, qBitMetaData, testQBitConfig, producers); + } + + + + /******************************************************************************* + ** Getter for testQBitConfig + *******************************************************************************/ + public TestQBitConfig getTestQBitConfig() + { + return (this.testQBitConfig); + } + + + + /******************************************************************************* + ** Setter for testQBitConfig + *******************************************************************************/ + public void setTestQBitConfig(TestQBitConfig testQBitConfig) + { + this.testQBitConfig = testQBitConfig; + } + + + + /******************************************************************************* + ** Fluent setter for testQBitConfig + *******************************************************************************/ + public TestQBitProducer withTestQBitConfig(TestQBitConfig testQBitConfig) + { + this.testQBitConfig = testQBitConfig; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java new file mode 100644 index 00000000..7660758c --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Meta Data Producer for OtherTable + *******************************************************************************/ +public class OtherTableMetaDataProducer extends QBitComponentMetaDataProducer +{ + public static final String NAME = "otherTable"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean isEnabled() + { + return (getQBitConfig().getOtherTableConfig().getDoProvideTable()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName(NAME) + .withPrimaryKeyField("id") + .withBackendName(getQBitConfig().getOtherTableConfig().getBackendName()) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + + return (qTableMetaData); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java new file mode 100644 index 00000000..3ef79554 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Meta Data Producer for SomeTable + *******************************************************************************/ +public class SomeTableMetaDataProducer extends QBitComponentMetaDataProducer +{ + public static final String NAME = "someTable"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean isEnabled() + { + return (getQBitConfig().getIsSomeTableEnabled()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName(NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + + return (qTableMetaData); + } + +} From 6fe04e65df78aa5356839a40dfa852cf150bdd98 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:26:10 -0600 Subject: [PATCH 37/67] Add getValueFromRecordOrOldRecord --- .../RecordCustomizerUtilityInterface.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java index fa91e9ab..53947578 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; @@ -160,4 +161,18 @@ public interface RecordCustomizerUtilityInterface return (oldRecordMap); } + + /*************************************************************************** + ** + ***************************************************************************/ + static T getValueFromRecordOrOldRecord(String fieldName, QRecord record, Serializable primaryKey, Optional> oldRecordMap) + { + T value = (T) record.getValue(fieldName); + if(value == null && primaryKey != null && oldRecordMap.isPresent() && oldRecordMap.get().containsKey(primaryKey)) + { + value = (T) oldRecordMap.get().get(primaryKey).getValue(fieldName); + } + return value; + } + } From 46a1a77d1b61730cb7146de3cf323a74f510c6b4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:26:19 -0600 Subject: [PATCH 38/67] Add method getProcess --- .../model/actions/processes/RunBackendStepInput.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 84da02f7..ca066eb3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface; @@ -617,4 +618,14 @@ public class RunBackendStepInput extends AbstractActionInput } } } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public QProcessMetaData getProcess() + { + return (QContext.getQInstance().getProcess(getProcessName())); + } } From 2b9181b22ec251ba52560809b11e97764f80e701 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:26:54 -0600 Subject: [PATCH 39/67] Remove block that was adding fileName to requestedPath, idk, wasn't good --- .../actions/AbstractBaseFilesystemAction.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 30999153..db3d3d05 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -62,7 +62,6 @@ import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.Back import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; @@ -288,21 +287,8 @@ public abstract class AbstractBaseFilesystemAction QueryOutput queryOutput = new QueryOutput(queryInput); - String requestedPath = null; - QQueryFilter filter = queryInput.getFilter(); - if(filter != null && tableDetails.getCardinality().equals(Cardinality.ONE)) - { - if(filter.getCriteria() != null && filter.getCriteria().size() == 1) - { - QFilterCriteria criteria = filter.getCriteria().get(0); - if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) - { - requestedPath = ValueUtils.getValueAsString(criteria.getValues().get(0)); - } - } - } - - List files = listFiles(table, queryInput.getBackend(), requestedPath); + String requestedPath = null; + List files = listFiles(table, queryInput.getBackend(), requestedPath); switch(tableDetails.getCardinality()) { From cddc42db5b7c506ae8325d81b758ebc37698d0fb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:27:17 -0600 Subject: [PATCH 40/67] add testSimpleQueryForOneFile --- .../sftp/actions/SFTPQueryActionTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java index 092ea987..35f87644 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -28,6 +28,9 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -58,6 +61,20 @@ class SFTPQueryActionTest extends BaseSFTPTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSimpleQueryForOneFile() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "testfile-1.txt"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + /******************************************************************************* ** *******************************************************************************/ From 35c4049174e15d5525dbc1cac2ba3ec39a133ebe Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 10:23:02 -0600 Subject: [PATCH 41/67] Add LinkValues.TO_RECORD_FROM_TABLE_DYNAMIC and FileDownloadValues.DOWNLOAD_URL_DYNAMIC --- .../model/metadata/fields/AdornmentType.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java index 29982aad..9fb76f41 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java @@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields; import java.io.Serializable; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Objects; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; @@ -48,14 +52,14 @@ public enum AdornmentType ////////////////////////////////////////////////////////////////////////// - /******************************************************************************* ** *******************************************************************************/ public interface LinkValues { - String TARGET = "target"; - String TO_RECORD_FROM_TABLE = "toRecordFromTable"; + String TARGET = "target"; + String TO_RECORD_FROM_TABLE = "toRecordFromTable"; + String TO_RECORD_FROM_TABLE_DYNAMIC = "toRecordFromTableDynamic"; } @@ -72,6 +76,8 @@ public enum AdornmentType String SUPPLEMENTAL_PROCESS_NAME = "supplementalProcessName"; String SUPPLEMENTAL_CODE_REFERENCE = "supplementalCodeReference"; + String DOWNLOAD_URL_DYNAMIC = "downloadUrlDynamic"; + //////////////////////////////////////////////////// // use these two together, as in: // // FILE_NAME_FORMAT = "Order %s Packing Slip.pdf" // @@ -79,6 +85,17 @@ public enum AdornmentType //////////////////////////////////////////////////// String FILE_NAME_FORMAT = "fileNameFormat"; String FILE_NAME_FORMAT_FIELDS = "fileNameFormatFields"; + + /*************************************************************************** + ** + ***************************************************************************/ + static String makeFieldDownloadUrl(String tableName, Serializable primaryKey, String fieldName, String fileName) + { + return ("/data/" + tableName + "/" + + URLEncoder.encode(Objects.requireNonNullElse(ValueUtils.getValueAsString(primaryKey), ""), StandardCharsets.UTF_8) + "/" + + fieldName + "/" + + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8)); + } } From 80c286ab00ae47d859fc7873e6773598db6c56d8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 10:46:39 -0600 Subject: [PATCH 42/67] update setBlobValuesToDownloadUrls to not do that if the field is set to use a downloadUrlDyanmic. --- .../core/actions/values/QValueFormatter.java | 16 +-- .../actions/values/QValueFormatterTest.java | 102 ++++++++++++++++++ 2 files changed, 111 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 6129b409..876b76ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -23,8 +23,6 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -34,7 +32,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -48,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -488,6 +486,8 @@ public class QValueFormatter String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT)); String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION)); + Boolean downloadUrlDynamic = ValueUtils.getValueAsBoolean(adornmentValues.get(AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC)); + for(QRecord record : records) { if(!doesFieldHaveValue(field, record)) @@ -495,6 +495,11 @@ public class QValueFormatter continue; } + if(BooleanUtils.isTrue(downloadUrlDynamic)) + { + continue; + } + Serializable primaryKey = record.getValue(table.getPrimaryKeyField()); String fileName = null; @@ -544,10 +549,7 @@ public class QValueFormatter || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE) || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME)) { - record.setValue(field.getName(), "/data/" + table.getName() + "/" - + URLEncoder.encode(ValueUtils.getValueAsString(primaryKey), StandardCharsets.UTF_8) + "/" - + field.getName() + "/" - + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8)); + record.setValue(field.getName(), AdornmentType.FileDownloadValues.makeFieldDownloadUrl(table.getName(), primaryKey, field.getName(), fileName)); } record.setDisplayValue(field.getName(), fileName); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java index cd678974..c4eac300 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java @@ -30,19 +30,23 @@ import java.time.LocalTime; import java.time.Month; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.DateTimeDisplayValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -237,4 +241,102 @@ class QValueFormatterTest extends BaseTest assertEquals("2024-04-04 02:12:00 PM CDT", record.getDisplayValue("createDate")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBlobValuesToDownloadUrls() + { + byte[] blobBytes = "hello".getBytes(); + { + QTableMetaData table = new QTableMetaData() + .withName("testTable") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("blobField", QFieldType.BLOB) + .withFieldAdornment(new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "blob-%s.txt") + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS, new ArrayList<>(List.of("id"))))); + + ////////////////////////////////////////////////////////////////// + // verify display value gets set to formated file-name + fields // + // and raw value becomes URL for downloading the byte // + ////////////////////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/blob-47.txt", record.getValueString("blobField")); + assertEquals("blob-47.txt", record.getDisplayValue("blobField")); + + //////////////////////////////////////////////////////// + // verify that w/ no blob value, we don't do anything // + //////////////////////////////////////////////////////// + QRecord recordWithoutBlobValue = new QRecord().withValue("id", 47); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(recordWithoutBlobValue)); + assertNull(recordWithoutBlobValue.getValue("blobField")); + assertNull(recordWithoutBlobValue.getDisplayValue("blobField")); + } + + { + FieldAdornment adornment = new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName"); + + QTableMetaData table = new QTableMetaData() + .withName("testTable") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("blobField", QFieldType.BLOB) + .withFieldAdornment(adornment)); + + //////////////////////////////////////////////////// + // here get the file name directly from one field // + //////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes).withValue("fileName", "myBlob.txt"); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/myBlob.txt", record.getValueString("blobField")); + assertEquals("myBlob.txt", record.getDisplayValue("blobField")); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // switch to use dynamic url, rerun, and assert we get the values as they were on the record before the call // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + adornment.withValue(AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC, true); + record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes).withValue("fileName", "myBlob.txt") + .withDisplayValue("blobField:" + AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC, "/something-custom/") + .withDisplayValue("blobField", "myDisplayValue"); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertArrayEquals(blobBytes, record.getValueByteArray("blobField")); + assertEquals("myDisplayValue", record.getDisplayValue("blobField")); + } + + { + FieldAdornment adornment = new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD); + + QTableMetaData table = new QTableMetaData() + .withName("testTable") + .withLabel("Test Table") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("blobField", QFieldType.BLOB).withLabel("Blob").withFieldAdornment(adornment)); + + /////////////////////////////////////////////////////////////////////////////////////////// + // w/o file name format or whatever, generate a file name from table & id & field labels // + /////////////////////////////////////////////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/Test+Table+47+Blob", record.getValueString("blobField")); + assertEquals("Test Table 47 Blob", record.getDisplayValue("blobField")); + + //////////////////////////////////////// + // add a default extension and re-run // + //////////////////////////////////////// + adornment.withValue(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION, "html"); + record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/Test+Table+47+Blob.html", record.getValueString("blobField")); + assertEquals("Test Table 47 Blob.html", record.getDisplayValue("blobField")); + } + } + } \ No newline at end of file From 77cc2724259b7e9d3822278cb2a74adadda768ac Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 11:07:22 -0600 Subject: [PATCH 43/67] Initial checkin --- .../widgets/RecordListWidgetRenderer.java | 251 ++++++++++++++++++ .../widgets/RecordListWidgetRendererTest.java | 188 +++++++++++++ 2 files changed, 439 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java new file mode 100644 index 00000000..ffd6710e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java @@ -0,0 +1,251 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.AbstractWidgetMetaDataBuilder; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Generic widget to display a list of records. + ** + ** Note, closely related to (and copied from ChildRecordListRenderer. + ** opportunity to share more code with that in the future?? + *******************************************************************************/ +public class RecordListWidgetRenderer extends AbstractWidgetRenderer +{ + private static final QLogger LOG = QLogger.getLogger(RecordListWidgetRenderer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Builder widgetMetaDataBuilder(String widgetName) + { + return (new Builder(new QWidgetMetaData() + .withName(widgetName) + .withIsCard(true) + .withCodeReference(new QCodeReference(RecordListWidgetRenderer.class)) + .withType(WidgetType.CHILD_RECORD_LIST.getType()) + .withValidatorPlugin(new RecordListWidgetValidator()) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class Builder extends AbstractWidgetMetaDataBuilder + { + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Builder(QWidgetMetaData widgetMetaData) + { + super(widgetMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withLabel(String label) + { + widgetMetaData.setLabel(label); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withMaxRows(Integer maxRows) + { + widgetMetaData.withDefaultValue("maxRows", maxRows); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withTableName(String tableName) + { + widgetMetaData.withDefaultValue("tableName", tableName); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withFilter(QQueryFilter filter) + { + widgetMetaData.withDefaultValue("filter", filter); + return (this); + } + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + try + { + Integer maxRows = null; + if(StringUtils.hasContent(input.getQueryParams().get("maxRows"))) + { + maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows")); + } + else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")) + { + maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().get("maxRows")); + } + + QQueryFilter filter = ((QQueryFilter) input.getWidgetMetaData().getDefaultValues().get("filter")).clone(); + filter.interpretValues(new HashMap<>(input.getQueryParams()), FilterUseCase.DEFAULT); + filter.setLimit(maxRows); + + String tableName = ValueUtils.getValueAsString(input.getWidgetMetaData().getDefaultValues().get("tableName")); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(tableName); + queryInput.setShouldTranslatePossibleValues(true); + queryInput.setShouldGenerateDisplayValues(true); + queryInput.setFilter(filter); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + QValueFormatter.setBlobValuesToDownloadUrls(table, queryOutput.getRecords()); + + int totalRows = queryOutput.getRecords().size(); + if(maxRows != null && (queryOutput.getRecords().size() == maxRows)) + { + ///////////////////////////////////////////////////////////////////////////////////// + // if the input said to only do some max, and the # of results we got is that max, // + // then do a count query, for displaying 1-n of // + ///////////////////////////////////////////////////////////////////////////////////// + CountInput countInput = new CountInput(); + countInput.setTableName(tableName); + countInput.setFilter(filter); + totalRows = new CountAction().execute(countInput).getCount(); + } + + String tablePath = QContext.getQInstance().getTablePath(tableName); + String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset())); + + ChildRecordListData widgetData = new ChildRecordListData(input.getQueryParams().get("widgetLabel"), queryOutput, table, tablePath, viewAllLink, totalRows); + + return (new RenderWidgetOutput(widgetData)); + } + catch(Exception e) + { + LOG.warn("Error rendering record list widget", e, logPair("widgetName", () -> input.getWidgetMetaData().getName())); + throw (e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static class RecordListWidgetValidator implements QInstanceValidatorPluginInterface + { + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void validate(QWidgetMetaDataInterface widgetMetaData, QInstance qInstance, QInstanceValidator qInstanceValidator) + { + String prefix = "Widget " + widgetMetaData.getName() + ": "; + + ////////////////////////////////////////////// + // make sure table name is given and exists // + ////////////////////////////////////////////// + QTableMetaData table = null; + String tableName = ValueUtils.getValueAsString(CollectionUtils.nonNullMap(widgetMetaData.getDefaultValues()).get("tableName")); + if(qInstanceValidator.assertCondition(StringUtils.hasContent(tableName), prefix + "defaultValue for tableName must be given")) + { + //////////////////////////// + // make sure table exists // + //////////////////////////// + table = qInstance.getTable(tableName); + qInstanceValidator.assertCondition(table != null, prefix + "No table named " + tableName + " exists in the instance"); + } + + //////////////////////////////////////////////////////////////////////////////////// + // make sure filter is given and is valid (only check that if table is given too) // + //////////////////////////////////////////////////////////////////////////////////// + QQueryFilter filter = ((QQueryFilter) widgetMetaData.getDefaultValues().get("filter")); + if(qInstanceValidator.assertCondition(filter != null, prefix + "defaultValue for filter must be given") && table != null) + { + qInstanceValidator.validateQueryFilter(qInstance, prefix, table, filter, null); + } + } + + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java new file mode 100644 index 00000000..64246384 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for RecordListWidgetRenderer + *******************************************************************************/ +class RecordListWidgetRendererTest extends BaseTest +{ + + /*************************************************************************** + ** + ***************************************************************************/ + private QWidgetMetaData defineWidget() + { + return RecordListWidgetRenderer.widgetMetaDataBuilder("testRecordListWidget") + .withTableName(TestUtils.TABLE_NAME_SHAPE) + .withMaxRows(20) + .withLabel("Some Shapes") + .withFilter(new QQueryFilter() + .withCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, "${input.maxShapeId}") + .withCriteria("name", QCriteriaOperator.NOT_EQUALS, "Square") + .withOrderBy(new QFilterOrderBy("id", false)) + ).getWidgetMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidation() throws QInstanceValidationException + { + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + widgetMetaData.getDefaultValues().remove("tableName"); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("defaultValue for tableName must be given"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + widgetMetaData.getDefaultValues().remove("filter"); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("defaultValue for filter must be given"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + widgetMetaData.getDefaultValues().remove("tableName"); + widgetMetaData.getDefaultValues().remove("filter"); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("defaultValue for filter must be given") + .hasMessageContaining("defaultValue for tableName must be given"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + QQueryFilter filter = (QQueryFilter) widgetMetaData.getDefaultValues().get("filter"); + filter.addCriteria(new QFilterCriteria("noField", QCriteriaOperator.EQUALS, "noValue")); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("Criteria fieldName noField is not a field in this table"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + qInstance.addWidget(widgetMetaData); + + ////////////////////////////////// + // make sure valid setup passes // + ////////////////////////////////// + new QInstanceValidator().validate(qInstance); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRender() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + qInstance.addWidget(widgetMetaData); + + TestUtils.insertDefaultShapes(qInstance); + TestUtils.insertExtraShapes(qInstance); + + { + RecordListWidgetRenderer recordListWidgetRenderer = new RecordListWidgetRenderer(); + RenderWidgetInput input = new RenderWidgetInput(); + input.setWidgetMetaData(widgetMetaData); + input.setQueryParams(Map.of("maxShapeId", "1")); + RenderWidgetOutput output = recordListWidgetRenderer.render(input); + + ChildRecordListData widgetData = (ChildRecordListData) output.getWidgetData(); + assertEquals(1, widgetData.getTotalRows()); + assertEquals(1, widgetData.getQueryOutput().getRecords().get(0).getValue("id")); + assertEquals("Triangle", widgetData.getQueryOutput().getRecords().get(0).getValue("name")); + } + + { + RecordListWidgetRenderer recordListWidgetRenderer = new RecordListWidgetRenderer(); + RenderWidgetInput input = new RenderWidgetInput(); + input.setWidgetMetaData(widgetMetaData); + input.setQueryParams(Map.of("maxShapeId", "4")); + RenderWidgetOutput output = recordListWidgetRenderer.render(input); + + ChildRecordListData widgetData = (ChildRecordListData) output.getWidgetData(); + assertEquals(3, widgetData.getTotalRows()); + + ///////////////////////////////////////////////////////////////////////// + // id=2,name=Square was skipped due to NOT_EQUALS Square in the filter // + // max-shape-id applied we don't get id=5 or 6 // + // and they're ordered as specified in the filter (id desc) // + ///////////////////////////////////////////////////////////////////////// + assertEquals(4, widgetData.getQueryOutput().getRecords().get(0).getValue("id")); + assertEquals("Rectangle", widgetData.getQueryOutput().getRecords().get(0).getValue("name")); + + assertEquals(3, widgetData.getQueryOutput().getRecords().get(1).getValue("id")); + assertEquals("Circle", widgetData.getQueryOutput().getRecords().get(1).getValue("name")); + + assertEquals(1, widgetData.getQueryOutput().getRecords().get(2).getValue("id")); + assertEquals("Triangle", widgetData.getQueryOutput().getRecords().get(2).getValue("name")); + } + } + +} \ No newline at end of file From a0d12eade7389eab69d03e5f7a0ed102cf0df908 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 11:14:56 -0600 Subject: [PATCH 44/67] Make validateQueryFilter public --- .../qqq/backend/core/instances/QInstanceValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index a214ee6b..b5be0c57 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -1905,7 +1905,7 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List queryJoins) + public void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List queryJoins) { for(QFilterCriteria criterion : CollectionUtils.nonNullList(queryFilter.getCriteria())) { From b984959aa736bdcf4ce245532061e5255869c4fa Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 14:25:30 -0600 Subject: [PATCH 45/67] A little more flexibility in filter validation, for context w/o a joinContext --- .../core/instances/QInstanceValidator.java | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index b5be0c57..b123858e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -142,6 +142,8 @@ public class QInstanceValidator private static ListingHash, QInstanceValidatorPluginInterface> validatorPlugins = new ListingHash<>(); + private JoinGraph joinGraph = null; + private List errors = new ArrayList<>(); @@ -169,8 +171,7 @@ public class QInstanceValidator // the enricher will build a join graph (if there are any joins). we'd like to only do that // // once, during the enrichment/validation work, so, capture it, and store it back in the instance. // ///////////////////////////////////////////////////////////////////////////////////////////////////// - JoinGraph joinGraph = null; - long start = System.currentTimeMillis(); + long start = System.currentTimeMillis(); try { ///////////////////////////////////////////////////////////////////////////////////////////////// @@ -179,7 +180,7 @@ public class QInstanceValidator // TODO - possible point of customization (use a different enricher, or none, or pass it options). QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(qInstance); qInstanceEnricher.enrich(); - joinGraph = qInstanceEnricher.getJoinGraph(); + this.joinGraph = qInstanceEnricher.getJoinGraph(); } catch(Exception e) { @@ -1949,7 +1950,8 @@ public class QInstanceValidator { if(fieldName.contains(".")) { - String fieldNameAfterDot = fieldName.substring(fieldName.lastIndexOf(".") + 1); + String fieldNameAfterDot = fieldName.substring(fieldName.lastIndexOf(".") + 1); + String tableNameBeforeDot = fieldName.substring(0, fieldName.lastIndexOf(".")); if(CollectionUtils.nullSafeHasContents(queryJoins)) { @@ -1973,11 +1975,32 @@ public class QInstanceValidator } else { - errors.add("QInstanceValidator does not yet support finding a field that looks like a join field, but isn't associated with a query."); - return (true); - // todo! for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values()) - // { - // } + if(this.joinGraph != null) + { + Set joinConnections = joinGraph.getJoinConnections(table.getName()); + for(JoinGraph.JoinConnectionList joinConnectionList : joinConnections) + { + JoinGraph.JoinConnection joinConnection = joinConnectionList.list().get(joinConnectionList.list().size() - 1); + if(tableNameBeforeDot.equals(joinConnection.joinTable())) + { + QTableMetaData joinTable = qInstance.getTable(tableNameBeforeDot); + if(joinTable.getFields().containsKey(fieldNameAfterDot)) + { + ///////////////////////// + // mmm, looks valid... // + ///////////////////////// + return (true); + } + } + } + } + + ////////////////////////////////////////////////////////////////////////////////////// + // todo - not sure how vulnerable we are to ongoing issues here... // + // idea: let a filter (or any object?) be opted out of validation, some version of // + // a static map of objects we can check at the top of various validate methods... // + ////////////////////////////////////////////////////////////////////////////////////// + errors.add("Failed to find field named: " + fieldName); } } } From 21c4434831646be3c17c6b6b8e86c6d40dfb68e8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 19:57:07 -0600 Subject: [PATCH 46/67] Add support for public-key based authentication --- .../sftp/actions/AbstractSFTPAction.java | 52 +++++++++++++++---- .../actions/SFTPTestConnectionAction.java | 36 ++++++++++++- .../model/metadata/SFTPBackendMetaData.java | 32 ++++++++++++ .../metadata/SFTPBackendVariantSetting.java | 3 +- .../module/filesystem/sftp/BaseSFTPTest.java | 11 ++++ .../actions/SFTPTestConnectionActionTest.java | 33 ++++++++++++ .../src/test/resources/README.md | 11 ++++ .../src/test/resources/test-only-key | 52 +++++++++++++++++++ .../src/test/resources/test-only-key.pub | 1 + 9 files changed, 219 insertions(+), 12 deletions(-) create mode 100644 qqq-backend-module-filesystem/src/test/resources/README.md create mode 100644 qqq-backend-module-filesystem/src/test/resources/test-only-key create mode 100644 qqq-backend-module-filesystem/src/test/resources/test-only-key.pub diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 1ef137af..64a77632 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -26,6 +26,11 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.spec.PKCS8EncodedKeySpec; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -48,6 +53,7 @@ import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBacke import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; import org.apache.sshd.client.SshClient; import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.common.config.keys.KeyUtils; import org.apache.sshd.sftp.client.SftpClient; import org.apache.sshd.sftp.client.SftpClientFactory; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -60,6 +66,8 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction 0) + { + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(privateKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + PrivateKey privateKey = keyFactory.generatePrivate(keySpec); + PublicKey publicKey = KeyUtils.recoverPublicKey(privateKey); + clientSession.addPublicKeyIdentity(new KeyPair(publicKey, privateKey)); + } + + ////////////////////////////////////////////////// + // if we have a password, add password identity // + ////////////////////////////////////////////////// + if(StringUtils.hasContent(password)) + { + clientSession.addPasswordIdentity(password); + } + clientSession.auth().verify(); this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java index 53b93ccf..d31fbcbe 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java @@ -37,7 +37,7 @@ public class SFTPTestConnectionAction extends AbstractSFTPAction ***************************************************************************/ public SFTPTestConnectionTestOutput testConnection(SFTPTestConnectionTestInput input) { - try(SftpClient sftpClient = super.makeConnection(input.getUsername(), input.getHostName(), input.getPort(), input.getPassword())) + try(SftpClient sftpClient = super.makeConnection(input.getUsername(), input.getHostName(), input.getPort(), input.getPassword(), input.getPrivateKey())) { SFTPTestConnectionTestOutput output = new SFTPTestConnectionTestOutput().withIsConnectionSuccess(true); @@ -80,6 +80,7 @@ public class SFTPTestConnectionAction extends AbstractSFTPAction private Integer port; private String password; private String basePath; + private byte[] privateKey; @@ -251,6 +252,39 @@ public class SFTPTestConnectionAction extends AbstractSFTPAction return (this); } + + + /******************************************************************************* + ** Getter for privateKey + ** + *******************************************************************************/ + public byte[] getPrivateKey() + { + return privateKey; + } + + + + /******************************************************************************* + ** Setter for privateKey + ** + *******************************************************************************/ + public void setPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + } + + + + /******************************************************************************* + ** Fluent setter for privateKey + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + return (this); + } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java index c423f99e..28ed667f 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java @@ -34,6 +34,7 @@ public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData private String username; private String password; private String hostName; + private byte[] privateKey; private Integer port; @@ -195,4 +196,35 @@ public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData return (this); } + + + /******************************************************************************* + ** Getter for privateKey + *******************************************************************************/ + public byte[] getPrivateKey() + { + return (this.privateKey); + } + + + + /******************************************************************************* + ** Setter for privateKey + *******************************************************************************/ + public void setPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + } + + + + /******************************************************************************* + ** Fluent setter for privateKey + *******************************************************************************/ + public SFTPBackendMetaData withPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + return (this); + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java index b5205cb9..2a702fc8 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java @@ -34,5 +34,6 @@ public enum SFTPBackendVariantSetting implements BackendVariantSetting PASSWORD, HOSTNAME, PORT, - BASE_PATH + BASE_PATH, + PRIVATE_KEY } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java index 6c71ea42..e7093402 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java @@ -67,6 +67,17 @@ public class BaseSFTPTest extends BaseTest grantUploadFilesDirWritePermission(); + /////////////////////////////////////////////////// + // add our test-only public key to the container // + /////////////////////////////////////////////////// + String sshDir = "/home/" + USERNAME + "/.ssh"; + sftpContainer.execInContainer("mkdir", sshDir); + sftpContainer.execInContainer("chmod", "700", sshDir); + sftpContainer.execInContainer("chown", USERNAME, sshDir); + copyFileToContainer("test-only-key.pub", sshDir + "/authorized_keys"); + sftpContainer.execInContainer("chmod", "600", sshDir + "/authorized_keys"); + sftpContainer.execInContainer("chown", USERNAME, sshDir + "/authorized_keys"); + currentPort = sftpContainer.getMappedPort(22); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java index b1439cbf..5b023735 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java @@ -22,7 +22,12 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -157,6 +162,7 @@ class SFTPTestConnectionActionTest extends BaseSFTPTest } + /******************************************************************************* ** *******************************************************************************/ @@ -175,4 +181,31 @@ class SFTPTestConnectionActionTest extends BaseSFTPTest assertNull(output.getListBasePathErrorMessage()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testConnectViaPublicKey() throws Exception + { + try(InputStream resourceAsStream = getClass().getResourceAsStream("/test-only-key")) + { + String pem = IOUtils.readLines(resourceAsStream, StandardCharsets.UTF_8).stream() + .filter(s -> !s.startsWith("----")) + .collect(Collectors.joining("")); + + byte[] privateKeyBytes = Base64.getDecoder().decode(pem); + + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPrivateKey(privateKeyBytes) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + } + } + } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/README.md b/qqq-backend-module-filesystem/src/test/resources/README.md new file mode 100644 index 00000000..abaabf7a --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/README.md @@ -0,0 +1,11 @@ +The `test-only-key` / `test-only-key.pub` key pair in this directory was generated via: + +```shell +ssh-keygen -t rsa -b 4096 -m PEM -f test-only-key +openssl pkcs8 -topk8 -inform PEM -in test-only-key -outform PEM -nocrypt -out test-only-key-kpcs8.pem +cp test-only-key-kpcs8.pem .../src/test/resources/test-only-key +``` + +It is NOT meant to be used as a secure key in ANY environment. + +It is included in this repo ONLY to be used for basic unit testing. \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/test-only-key b/qqq-backend-module-filesystem/src/test/resources/test-only-key new file mode 100644 index 00000000..c3ca2a4d --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/test-only-key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDGEYxHZMPM6A+4 +OtayotUrB94AqYow1PcK5HzY8psKuHYejurugwFBbrFHq+dOso6EkLKcSpjvEX+X +Se+m3CDIC6qVuLBS6fmWe6wo73fhlqyEP6rslz//v5+Di8fhCPSfa8y4X5RLbkDR +yrL/ilmIEHUcmCU82SFFp9C10W+Ir9SZbjAoiRHT+X/WWmXYnMrfLHl38xxdWr8O +qXEmLaVxmSaLnWAmMXRr4Nqrk7r7VVq899CZSJMVLSfqeiJPqyLkwOPcYjzOKft0 +hYAhIXCUsEK4Wv7R1t9SooC7nqE2mY/htBcrPPyDhTT4Kl0zAD2MNloNUUQsTnic +t3FCUy0pB9zr26bZ4YHBIvdReTWcx6IlH2U28V9ktxWRJe/ESqay4RTfif24aWya +WzD9+yTowTdPJbMaQZI5XGYJBfFtXr8aC8Di+VuILbriiuLk1iPuj1p2R3JXepON +sVRDBm10SmHdETcsZXB3glqqiV+mFGmHguPXSo1WZmP79+kj/phwtn2wFKRLwi6k +QNeqOxLbamRtj7y4M2sphEC7Cih1w4J+CzjZE8PC7+Bck+ldvmdtW6+wkkhLgli9 +SvDwe2pM8eTWTznzs1Qj9My5p1v0uvIo0xADtpITzb3kQtl9+2C7uqscjJLIpc0g +VM5q55bFgxUp32G+HKUT8TI98ZrbEwIDAQABAoICAQC1f2MCEO3zGDs/YHtYmimo +Er917+W3tY7jJljZHAbCniFvAxt4kAdYhCxjNrzwumIqS8W/vgPCHlDCu3eleVV4 +umgIZoL8l3akVJN/t2AtEbroPMdNoZN9sYRkMHRqW6B9bXTdBoiHTnKLS6kWzRoZ +uqr2Ft0YkwcQIyT3VwFTSXwRVI1At8nkal6gd5mYEqU8OC7eoaG+Ued9cftDNtTB +8csGaKGwneTG7fay/t56bM6HFrbJn11YLFbFYEGMq49/+tlSG5sIeCP5tFOjCFd3 +iMS61ndmpfVibZJ1Wnjz5WeZNUN91Za2lhvhxEA++dtsXmnKhktoJkgTo21fj4Ry +1DbdysR5W0vz+HQ5usT/fWAypKj4KSlP6rsL/zkqmlxW1qE/nwzK6UN6kVWvO1Sm +6EcZIQKTRpkGFXluOJL/Bkqg+4Ayl9G3veLON5yTOB367m1haI/oIgr+jOKrkzdG +PWWNZ78USCT9YD/P2vHkwVQ/uzF3kHhLL6n8hzunFpJk+5eScXsyXTUjDCn4HurE +A8Z1FHLOeu4EEKODEdg1PCa34/8Z0K+88G5G/sTDWLf2MjywdCSDzKxYiSpQW0gt +5I5WgQG8CiGHbysBWMnqjoG1J+QpAUDzT29CuLhVbn4i0uiTe0AZ0xLjravhd5dW +dlItzvOY1rj5jHPs5hiWkQKCAQEA97jAuCzPw2Ats2QUfpnqEd/ITKRYTtK6zo+c +sdc1JH0RnzxCeBr2BApSrfMbqAMrPaiUKk5FUqpF5q47AXGrYFlaAPe7cq+Lr97N +LRi7NHM9RdF1myollCghwgKkq8qUe64eNAypMtGOdjrMh7kC879udESYXLjqgT5H +wQbHF5IRvg5vBkVKxzLrg37lX1f0MVGdB8VxA+QRQ4egFuQApPvMLFVenYZL165r +u3OIpOQWcJ91L5VzN5jJMR1x8VWFR6iD4PxachdmD8qaNxfwWKTTWxkpwTXfW1EI +68NZ2s9RmuRbJEfOtiznfzdVL+lAibeMe7dvUMzIevr0hPzMNwKCAQEAzLADz691 +9bJniRVFUkHlbXRV7k0sRxAXNzGmar8xpnG3wJaW+zSsj8Nnr+aSBGgGWmmldEn7 +tiHlozrjWUgUEsRJKObiCrdGsTC7+0flQLs1bukLfgZSnP3oMDYAgq15Vaw49oCU +M4KxRGfkPEhwP/DZClHsYkPr6HegT2/21z8AFHTAxknGjWWGJJAFxwaog7Akugdy +gXLb7lU4SjCJdb5tR1c7aDEUOvDDu1iffhWtt5Tp9BL0dKlN4M+6XZvqSNiVlN6P +BB5gDuSa0qEewIbMWiT4rcvE7gCSXFEWPnGbtHU7QcI4Wx1F75Y4CRgs2rRlnj9j +bVAsRNIOTqkSBQKCAQEAliEL+xJ9X6TcTYnrucZBy09aLsizFCI2QJVcm5MXi+OY +WG7Gwc9lJZG0BeP98Nbqz9Vo5jLFZJH5BxK0g+2FtUCxgUCiA6FMAOwAYMJKQkFM +8xE8OytR1vZzbwb3EX4WetZNS7IYoMnLku+ToPWJSnvLzv77b8ZJqMY76knXQvut +cQeCVcSMyyia/vhavmupfHI/vsPz+C2yIMEDTpwjn9lSJdQfIUyQjkgQ1mvwdi4d +Q2gANzRVvW4FEJUNxvrTaVhBhIqrrdVsb0mUKKuDZ9WMmfsoCQZDNS5pP6kGvctD +Y6HdcqFqL5ILQlggcobkLBJnO1syRT+2iIGqyyYCBQKCAQAbwy/xJnJQbe8/F6R8 +YLW2n9Xb6Zm81cDgWpqg1eftFHWA6Kv3zJAvO6i/of1iHZ3m+3dWi4ZZkMVt21nk +zTLzzK3Dn3U/UNaEyABnN7wviHTZ40AMyty/sGyixWBSWScg6KgdPxla1zol9hVt +28Fl2swFa1EtjtrbgAY9YAlR7pibLa7L9ku49/E22lX+RbfrjKOem837ItITxHlL +DsRGNRrrVziWjDmbOPbDXWTcnCIgyVDmKv//JsuKV4KGmdQwJzg6pekt/NS4kGcz +dGkQYfgrreIQ6JeAVJGFdfYXaB9fXZs48xfju9e1hGF7Uk0bKOazjRN2Sy6F8xu/ +rYzlAoIBAEjY7u3Jmntn1AYsbuy9wTblKl1IaZP4ST+X0/dLtvW8ZLsx0jPGwMXx +xmOku5OGqPjCn5i8Ws2KS8O6O+7lGm/CHXvmDpozD3wpjnJ64SgoLnjrT8R78TEJ +UjsGQfR7ofSj4heR7TgEPp+n0SXse3qERd6VZ5YPuzGva1iVJogErwI58QU2QaxQ +0ONV6F8oZuXjUs9KRhXQ8W0i87m0P7/ZumhqPaQqY/MeAYF/ED5C6ETKISxaDqs/ +zd/jf6uPZL6P4DPWcw7cSk5/aNZZ0P+/BkEX33WHBDSdVyHC+ydMcYZBrrlWKoSt +sNTITZbKrQB4hwHdawpMHxh+5mRXLk0= +-----END PRIVATE KEY----- diff --git a/qqq-backend-module-filesystem/src/test/resources/test-only-key.pub b/qqq-backend-module-filesystem/src/test/resources/test-only-key.pub new file mode 100644 index 00000000..02a25a74 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/test-only-key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDGEYxHZMPM6A+4OtayotUrB94AqYow1PcK5HzY8psKuHYejurugwFBbrFHq+dOso6EkLKcSpjvEX+XSe+m3CDIC6qVuLBS6fmWe6wo73fhlqyEP6rslz//v5+Di8fhCPSfa8y4X5RLbkDRyrL/ilmIEHUcmCU82SFFp9C10W+Ir9SZbjAoiRHT+X/WWmXYnMrfLHl38xxdWr8OqXEmLaVxmSaLnWAmMXRr4Nqrk7r7VVq899CZSJMVLSfqeiJPqyLkwOPcYjzOKft0hYAhIXCUsEK4Wv7R1t9SooC7nqE2mY/htBcrPPyDhTT4Kl0zAD2MNloNUUQsTnict3FCUy0pB9zr26bZ4YHBIvdReTWcx6IlH2U28V9ktxWRJe/ESqay4RTfif24aWyaWzD9+yTowTdPJbMaQZI5XGYJBfFtXr8aC8Di+VuILbriiuLk1iPuj1p2R3JXepONsVRDBm10SmHdETcsZXB3glqqiV+mFGmHguPXSo1WZmP79+kj/phwtn2wFKRLwi6kQNeqOxLbamRtj7y4M2sphEC7Cih1w4J+CzjZE8PC7+Bck+ldvmdtW6+wkkhLgli9SvDwe2pM8eTWTznzs1Qj9My5p1v0uvIo0xADtpITzb3kQtl9+2C7uqscjJLIpc0gVM5q55bFgxUp32G+HKUT8TI98ZrbEw== test-only-key From cdc6df214048c71badca9d6eb59835a54eb9ad67 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 20:10:26 -0600 Subject: [PATCH 47/67] Removing call to remove all writeCapabilities from RenderedReport table... not entirely clear that's wanted anyway, and it's a change in behavior now, since this overload of withoutCapabilities was fixed... --- .../core/model/savedreports/SavedReportsMetaDataProvider.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java index 2b40bd86..13084a42 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java @@ -50,7 +50,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -351,8 +350,7 @@ public class SavedReportsMetaDataProvider .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedReportId", "renderedReportStatusId"))) .withSection(new QFieldSection("input", new QIcon().withName("input"), Tier.T2, List.of("userId", "reportFormat"))) .withSection(new QFieldSection("output", new QIcon().withName("output"), Tier.T2, List.of("jobUuid", "resultPath", "rowCount", "errorMessage", "startTime", "endTime"))) - .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) - .withoutCapabilities(Capability.allWriteCapabilities()); + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); table.getField("renderedReportStatusId").setAdornments(List.of(new FieldAdornment(AdornmentType.CHIP) .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.RUNNING.getId(), "pending", AdornmentType.ChipValues.COLOR_SECONDARY)) From eae24e3ebaac17ae55b2b05e7e28cfa9828eaa03 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 25 Feb 2025 08:45:48 -0600 Subject: [PATCH 48/67] Add method pemStringToDecodedBytes --- .../sftp/actions/AbstractSFTPAction.java | 16 ++++++++++++++++ .../actions/SFTPTestConnectionActionTest.java | 9 ++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 64a77632..f98582e8 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -33,6 +33,7 @@ import java.security.PublicKey; import java.security.spec.PKCS8EncodedKeySpec; import java.time.Instant; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -409,4 +410,19 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction !s.startsWith("----")) - .collect(Collectors.joining("")); - - byte[] privateKeyBytes = Base64.getDecoder().decode(pem); + byte[] privateKeyBytes = AbstractSFTPAction.pemStringToDecodedBytes(StringUtils.join("", IOUtils.readLines(resourceAsStream, StandardCharsets.UTF_8))); SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() .withUsername(BaseSFTPTest.USERNAME) From 4b585cde459b524d5debae0ffcf9d91421ad8e1d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 25 Feb 2025 11:47:09 -0600 Subject: [PATCH 49/67] convert paths starting with / to be ./ instead --- .../filesystem/sftp/actions/AbstractSFTPAction.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index f98582e8..93a5adaf 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -310,6 +310,16 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction rs = new ArrayList<>(); + ///////////////////////////////////////////////////////////////////////////////////// + // at least in some cases, listing / seems to be interpreted by the server as // + // a listing from the root of the system, not just the user's dir. so, converting // + // paths starting with / to instead be ./ is giving us better results. // + ///////////////////////////////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = "." + fullPath; + } + for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) { if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename())) From 366f5d96000ed0d84c03c0486ad8961e136d5923 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:47:11 -0600 Subject: [PATCH 50/67] Initial checkin --- .../backend/core/scheduler/CronDescriber.java | 344 ++++++++++++++++++ .../CronExpressionTooltipFieldBehavior.java | 90 +++++ .../core/scheduler/CronDescriberTest.java | 68 ++++ ...ronExpressionTooltipFieldBehaviorTest.java | 67 ++++ 4 files changed, 569 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java new file mode 100644 index 00000000..2b2a2c56 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java @@ -0,0 +1,344 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.text.ParseException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/******************************************************************************* + ** class to give a human-friendly descriptive string from a cron expression. + ** (written in half by my friend Mr. Chatty G) + *******************************************************************************/ +public class CronDescriber +{ + private static final Map DAY_OF_WEEK_MAP = new HashMap<>(); + private static final Map MONTH_MAP = new HashMap<>(); + + static + { + DAY_OF_WEEK_MAP.put("1", "Sunday"); + DAY_OF_WEEK_MAP.put("2", "Monday"); + DAY_OF_WEEK_MAP.put("3", "Tuesday"); + DAY_OF_WEEK_MAP.put("4", "Wednesday"); + DAY_OF_WEEK_MAP.put("5", "Thursday"); + DAY_OF_WEEK_MAP.put("6", "Friday"); + DAY_OF_WEEK_MAP.put("7", "Saturday"); + + //////////////////////////////// + // Quartz also allows SUN-SAT // + //////////////////////////////// + DAY_OF_WEEK_MAP.put("SUN", "Sunday"); + DAY_OF_WEEK_MAP.put("MON", "Monday"); + DAY_OF_WEEK_MAP.put("TUE", "Tuesday"); + DAY_OF_WEEK_MAP.put("WED", "Wednesday"); + DAY_OF_WEEK_MAP.put("THU", "Thursday"); + DAY_OF_WEEK_MAP.put("FRI", "Friday"); + DAY_OF_WEEK_MAP.put("SAT", "Saturday"); + + MONTH_MAP.put("1", "January"); + MONTH_MAP.put("2", "February"); + MONTH_MAP.put("3", "March"); + MONTH_MAP.put("4", "April"); + MONTH_MAP.put("5", "May"); + MONTH_MAP.put("6", "June"); + MONTH_MAP.put("7", "July"); + MONTH_MAP.put("8", "August"); + MONTH_MAP.put("9", "September"); + MONTH_MAP.put("10", "October"); + MONTH_MAP.put("11", "November"); + MONTH_MAP.put("12", "December"); + + //////////////////////////////// + // Quartz also allows JAN-DEC // + //////////////////////////////// + MONTH_MAP.put("JAN", "January"); + MONTH_MAP.put("FEB", "February"); + MONTH_MAP.put("MAR", "March"); + MONTH_MAP.put("APR", "April"); + MONTH_MAP.put("MAY", "May"); + MONTH_MAP.put("JUN", "June"); + MONTH_MAP.put("JUL", "July"); + MONTH_MAP.put("AUG", "August"); + MONTH_MAP.put("SEP", "September"); + MONTH_MAP.put("OCT", "October"); + MONTH_MAP.put("NOV", "November"); + MONTH_MAP.put("DEC", "December"); + } + + /*************************************************************************** + ** + ***************************************************************************/ + public static String getDescription(String cronExpression) throws ParseException + { + String[] parts = cronExpression.split("\\s+"); + if(parts.length < 6 || parts.length > 7) + { + throw new ParseException("Invalid cron expression: " + cronExpression, 0); + } + + String seconds = parts[0]; + String minutes = parts[1]; + String hours = parts[2]; + String dayOfMonth = parts[3]; + String month = parts[4]; + String dayOfWeek = parts[5]; + String year = parts.length == 7 ? parts[6] : "*"; + + StringBuilder description = new StringBuilder(); + + description.append("At "); + description.append(describeTime(seconds, minutes, hours)); + description.append(", on "); + description.append(describeDayOfMonth(dayOfMonth)); + description.append(" of "); + description.append(describeMonth(month)); + description.append(", "); + description.append(describeDayOfWeek(dayOfWeek)); + if(!year.equals("*")) + { + description.append(", in ").append(year); + } + description.append("."); + + return description.toString(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeTime(String seconds, String minutes, String hours) + { + return String.format("%s, %s, %s", describePart(seconds, "second"), describePart(minutes, "minute"), describePart(hours, "hour")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeDayOfMonth(String dayOfMonth) + { + if(dayOfMonth.equals("?")) + { + return "every day"; + } + else if(dayOfMonth.equals("L")) + { + return "the last day"; + } + else if(dayOfMonth.contains("W")) + { + return "the nearest weekday to day " + dayOfMonth.replace("W", ""); + } + else + { + return (describePart(dayOfMonth, "day")); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeMonth(String month) + { + if(month.equals("*")) + { + return "every month"; + } + else + { + String[] months = month.split(","); + StringBuilder result = new StringBuilder(); + for(String m : months) + { + result.append(MONTH_MAP.getOrDefault(m, m)).append(", "); + } + return result.substring(0, result.length() - 2); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeDayOfWeek(String dayOfWeek) + { + if(dayOfWeek.equals("?")) + { + return "every day of the week"; + } + else if(dayOfWeek.equals("L")) + { + return "the last day of the week"; + } + else if(dayOfWeek.contains("#")) + { + String[] parts = dayOfWeek.split("#"); + return String.format("the %s %s of the month", ordinal(parts[1]), DAY_OF_WEEK_MAP.getOrDefault(parts[0], parts[0])); + } + else if(dayOfWeek.contains("-")) + { + String[] parts = dayOfWeek.split("-"); + return String.format("from %s to %s", DAY_OF_WEEK_MAP.getOrDefault(parts[0], parts[0]), DAY_OF_WEEK_MAP.getOrDefault(parts[1], parts[1])); + } + else + { + String[] days = dayOfWeek.split(","); + StringBuilder result = new StringBuilder(); + for(String d : days) + { + result.append(DAY_OF_WEEK_MAP.getOrDefault(d, d)).append(", "); + } + return result.substring(0, result.length() - 2); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describePart(String part, String label) + { + if(part.equals("*")) + { + return "every " + label; + } + else if(part.contains("/")) + { + String[] parts = part.split("/"); + if(parts[0].equals("*")) + { + parts[0] = "0"; + } + return String.format("every %s " + label + "s starting at %s", parts[1], parts[0]); + } + else if(part.contains(",")) + { + if(label.equals("hour")) + { + String[] parts = part.split(","); + List partList = Arrays.stream(parts).map(p -> hourToAmPm(p)).toList(); + return String.join(", ", partList); + } + else + { + if(label.equals("day")) + { + return "days " + part.replace(",", ", "); + } + else + { + return part.replace(",", ", ") + " " + label + "s"; + } + } + } + else if(part.contains("-")) + { + String[] parts = part.split("-"); + if(label.equals("day")) + { + return String.format("%ss from %s to %s", label, parts[0], parts[1]); + } + else if(label.equals("hour")) + { + return String.format("from %s to %s", hourToAmPm(parts[0]), hourToAmPm(parts[1])); + } + else + { + return String.format("from %s to %s %s", parts[0], parts[1], label + "s"); + } + } + else + { + if(label.equals("day")) + { + return label + " " + part; + } + if(label.equals("hour")) + { + return hourToAmPm(part); + } + else + { + return part + " " + label + "s"; + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String hourToAmPm(String part) + { + try + { + int hour = Integer.parseInt(part); + return switch(hour) + { + case 0 -> "midnight"; + case 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 -> hour + " AM"; + case 12 -> "noon"; + case 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 -> (hour - 12) + " PM"; + default -> hour + " hours"; + }; + } + catch(Exception e) + { + return part + " hours"; + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String ordinal(String number) + { + int n = Integer.parseInt(number); + if(n >= 11 && n <= 13) + { + return n + "th"; + } + + return switch(n % 10) + { + case 1 -> n + "st"; + case 2 -> n + "nd"; + case 3 -> n + "rd"; + default -> n + "th"; + }; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java new file mode 100644 index 00000000..bd77f6b4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Field display behavior, to add a human-redable tooltip to cron-expressions. + *******************************************************************************/ +public class CronExpressionTooltipFieldBehavior implements FieldDisplayBehavior +{ + + /*************************************************************************** + ** Add both this behavior, and the tooltip adornment to a field + ** Note, if either was already there, then that part is left alone. + ***************************************************************************/ + public static void addToField(QFieldMetaData fieldMetaData) + { + CronExpressionTooltipFieldBehavior existingBehavior = fieldMetaData.getBehaviorOnlyIfSet(CronExpressionTooltipFieldBehavior.class); + if(existingBehavior == null) + { + fieldMetaData.withBehavior(new CronExpressionTooltipFieldBehavior()); + } + + if(fieldMetaData.getAdornment(AdornmentType.TOOLTIP).isEmpty()) + { + fieldMetaData.withFieldAdornment((new FieldAdornment(AdornmentType.TOOLTIP) + .withValue(AdornmentType.TooltipValues.TOOLTIP_DYNAMIC, true))); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : recordList) + { + try + { + String cronExpression = record.getValueString(field.getName()); + if(StringUtils.hasContent(cronExpression)) + { + String description = CronDescriber.getDescription(cronExpression); + record.setDisplayValue(field.getName() + ":" + AdornmentType.TooltipValues.TOOLTIP_DYNAMIC, description); + } + } + catch(Exception e) + { + ///////////////////// + // just leave null // + ///////////////////// + } + } + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java new file mode 100644 index 00000000..b5801d01 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.text.ParseException; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for CronDescriber + *******************************************************************************/ +class CronDescriberTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws ParseException + { + assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * * * ?")); + assertEquals("At 0 seconds, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 * * * * ?")); + assertEquals("At 0 seconds, 0 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 * * * ?")); + assertEquals("At 0 seconds, 0, 30 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0,30 * * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 0 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 1 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 1 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 11 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 11 * * ?")); + assertEquals("At 0 seconds, 0 minutes, noon, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 12 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 1 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 13 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 11 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 23 * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on day 10 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10 * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on days 10, 20 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10,20 * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on days from 10 to 15 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10-15 * ?")); + assertEquals("At from 10 to 15 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("10-15 0 0 * * ?")); + assertEquals("At 30 seconds, 30 minutes, from 8 AM to 4 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("30 30 8-16 * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every 3 days starting at 0 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 */3 * ?")); + assertEquals("At every 5 seconds starting at 0, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0/5 0 0 * * ?")); + assertEquals("At 0 seconds, every 30 minutes starting at 3, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 3/30 0 * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Monday, Wednesday, Friday.", CronDescriber.getDescription("0 0 0 * * MON,WED,FRI")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, from Monday to Friday.", CronDescriber.getDescription("0 0 0 * * MON-FRI")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday, Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7")); + assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?")); + assertEquals("??", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java new file mode 100644 index 00000000..26dc8879 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for CronExpressionTooltipFieldBehavior + *******************************************************************************/ +class CronExpressionTooltipFieldBehaviorTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QFieldMetaData field = new QFieldMetaData("cronExpression", QFieldType.STRING); + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE) + .addField(field); + + CronExpressionTooltipFieldBehavior.addToField(field); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SHAPE).withRecord( + new QRecord().withValue("name", "Square").withValue("cronExpression", "* * * * * ?"))); + + QRecord record = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_SHAPE).withPrimaryKey(1).withShouldGenerateDisplayValues(true)); + assertThat(record.getDisplayValue("cronExpression:" + AdornmentType.TooltipValues.TOOLTIP_DYNAMIC)) + .contains("every second"); + } + +} \ No newline at end of file From 27c816d627d5e6331c5031e447b2cdcee2901d86 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:53:42 -0600 Subject: [PATCH 51/67] Add a root exception --- .../qqq/backend/core/actions/reporting/CsvExportStreamer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java index fac33e13..d36305f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java @@ -108,7 +108,7 @@ public class CsvExportStreamer implements ExportStreamerInterface } catch(Exception e) { - throw (new QReportingException("Error starting CSV report")); + throw (new QReportingException("Error starting CSV report", e)); } } From 428832f4ec22b54c9baea41298a38e9074e60a2c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:54:00 -0600 Subject: [PATCH 52/67] Add discoverAndAddPluginsInPackage --- .../core/instances/QInstanceEnricher.java | 25 ++++++++++ .../core/instances/QInstanceEnricherTest.java | 35 ++++++++------ .../testplugins/TestEnricherPlugin.java | 48 +++++++++++++++++++ 3 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 809508cd..a234bfe1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -95,6 +95,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.utils.ClassPathUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -1482,6 +1483,30 @@ public class QInstanceEnricher + /*************************************************************************** + ** + ***************************************************************************/ + public static void discoverAndAddPluginsInPackage(String packageName) throws QException + { + try + { + for(Class aClass : ClassPathUtils.getClassesInPackage(packageName)) + { + if(QInstanceEnricherPluginInterface.class.isAssignableFrom(aClass)) + { + QInstanceEnricherPluginInterface plugin = (QInstanceEnricherPluginInterface) aClass.getConstructor().newInstance(); + addEnricherPlugin(plugin); + } + } + } + catch(Exception e) + { + throw (new QException("Error discovering and adding enricher plugins in package [" + packageName + "]", e)); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index 1817b57c..58e7ccca 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -27,7 +27,8 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.enrichment.testplugins.TestEnricherPlugin; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; @@ -595,27 +596,31 @@ class QInstanceEnricherTest extends BaseTest { QInstance qInstance = TestUtils.defineInstance(); - QInstanceEnricher.addEnricherPlugin(new QInstanceEnricherPluginInterface() - { - /*************************************************************************** - * - ***************************************************************************/ - @Override - public void enrich(QFieldMetaData field, QInstance qInstance) - { - if(field != null) - { - field.setLabel(field.getLabel() + " Plugged"); - } - } - }); + QInstanceEnricher.addEnricherPlugin(new TestEnricherPlugin()); new QInstanceEnricher(qInstance).enrich(); qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); qInstance.getProcesses().values().forEach(process -> process.getInputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); qInstance.getProcesses().values().forEach(process -> process.getOutputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDiscoverAndAddPlugins() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + new QInstanceEnricher(qInstance).enrich(); + qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).doesNotEndWith("Plugged"))); + + qInstance = TestUtils.defineInstance(); + QInstanceEnricher.discoverAndAddPluginsInPackage(getClass().getPackageName() + ".enrichment.testplugins"); + new QInstanceEnricher(qInstance).enrich(); + qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java new file mode 100644 index 00000000..9e940284 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.enrichment.testplugins; + + +import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestEnricherPlugin implements QInstanceEnricherPluginInterface +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void enrich(QFieldMetaData field, QInstance qInstance) + { + if(field != null) + { + field.setLabel(field.getLabel() + " Plugged"); + } + } + +} From 2703f06b2351f83246b01c2b5b54cf3d4cc21589 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:55:07 -0600 Subject: [PATCH 53/67] Add TOOLTIP type adornment; also, update url-encoding in FileDownload adornment to .replace("+", "%20") --- .../model/metadata/fields/AdornmentType.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java index 9fb76f41..4cf6e42b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java @@ -46,6 +46,7 @@ public enum AdornmentType REVEAL, FILE_DOWNLOAD, FILE_UPLOAD, + TOOLTIP, ERROR; ////////////////////////////////////////////////////////////////////////// // keep these values in sync with AdornmentType.ts in qqq-frontend-core // @@ -92,9 +93,9 @@ public enum AdornmentType static String makeFieldDownloadUrl(String tableName, Serializable primaryKey, String fieldName, String fileName) { return ("/data/" + tableName + "/" - + URLEncoder.encode(Objects.requireNonNullElse(ValueUtils.getValueAsString(primaryKey), ""), StandardCharsets.UTF_8) + "/" + + URLEncoder.encode(Objects.requireNonNullElse(ValueUtils.getValueAsString(primaryKey), ""), StandardCharsets.UTF_8).replace("+", "%20") + "/" + fieldName + "/" - + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8)); + + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8).replace("+", "%20")); } } @@ -246,4 +247,15 @@ public enum AdornmentType } } + + + /******************************************************************************* + ** + *******************************************************************************/ + public interface TooltipValues + { + String STATIC_TEXT = "staticText"; + String TOOLTIP_DYNAMIC = "tooltipDynamic"; + } + } From 1354755372aab7429949e85b52497811bb5dc854 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:56:05 -0600 Subject: [PATCH 54/67] Make some of hard-coded table & field names optionally come from widget input, for more flexible usage (e.g., by sftp-data-integration qbit's report export setup) --- ...ReportValuesDynamicFormWidgetRenderer.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java index 3a448eda..f71da271 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java @@ -56,7 +56,16 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* ** Note - exists under 2 names, for the RenderSavedReport process, and for the - ** ScheduledReport table + ** ScheduledReport table (and can be used in your custom code too: + * + ** by default, in qqq backend core, we'll assume this widget is being used on the + ** view screen for a ScheduledReport, with field names that we know from that table. + ** But, allow it to be used on a different table (optionally with different field names), + ** coming from the input map. + ** + ** e.g., that one may set in widget metaData as: + ** .withDefaultValue("tableName", "myTable") + ** .withDefaultValue("fieldNameId", "identifier"), etc. *******************************************************************************/ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRenderer { @@ -88,11 +97,16 @@ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRendere } else if(input.getQueryParams().containsKey("id")) { - QRecord scheduledReportRecord = new GetAction().executeForRecord(new GetInput(ScheduledReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get("id")))); - QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(scheduledReportRecord.getValueInteger("savedReportId")))); + String tableName = input.getQueryParams().getOrDefault("tableName", ScheduledReport.TABLE_NAME); + String fieldNameId = input.getQueryParams().getOrDefault("fieldNameId", "id"); + String fieldNameSavedReportId = input.getQueryParams().getOrDefault("fieldNameSavedReportId", "savedReportId"); + String fieldNameInputValues = input.getQueryParams().getOrDefault("fieldNameInputValues", "inputValues"); + + QRecord hostRecord = new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get(fieldNameId)))); + QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(hostRecord.getValueInteger(fieldNameSavedReportId)))); savedReport = new SavedReport(record); - String inputValues = scheduledReportRecord.getValueString("inputValues"); + String inputValues = hostRecord.getValueString(fieldNameInputValues); if(StringUtils.hasContent(inputValues)) { JSONObject jsonObject = JsonUtils.toJSONObject(inputValues); @@ -197,8 +211,8 @@ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRendere } catch(Exception e) { - LOG.warn("Error rendering scheduled report values dynamic form widget", e, logPair("queryParams", String.valueOf(input.getQueryParams()))); - throw (new QException("Error rendering scheduled report values dynamic form widget", e)); + LOG.warn("Error rendering report values dynamic form widget", e, logPair("queryParams", String.valueOf(input.getQueryParams()))); + throw (new QException("Error rendering report values dynamic form widget", e)); } } From b87fb6bd4a2ed65d89c4b4e5a6c09dfb244cb06e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:11:11 -0600 Subject: [PATCH 55/67] Adjust inserted-ids process summary line for when only 1 record was inserted --- .../bulk/insert/BulkInsertLoadStep.java | 5 + .../insert/BulkInsertFullProcessTest.java | 205 ++++++++++++------ 2 files changed, 147 insertions(+), 63 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java index b35bb24b..30f98c91 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -110,6 +111,10 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm if(field.getType().isNumeric()) { ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey); + if(Objects.equals(firstInsertedPrimaryKey, lastInsertedPrimaryKey)) + { + idsLine.setMessage("Inserted " + field.getLabel() + " " + firstInsertedPrimaryKey); + } idsLine.setCount(null); processSummary.add(idsLine); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index 2ab31159..91972a91 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -22,24 +22,21 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; -import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; @@ -61,6 +58,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; *******************************************************************************/ class BulkInsertFullProcessTest extends BaseTest { + private static final String defaultEmail = "noone@kingsrook.com"; + + /******************************************************************************* ** @@ -116,48 +116,20 @@ class BulkInsertFullProcessTest extends BaseTest @Test void test() throws Exception { - String defaultEmail = "noone@kingsrook.com"; - - /////////////////////////////////////// - // make sure table is empty to start // - /////////////////////////////////////// assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); - QInstance qInstance = QContext.getQInstance(); - String processName = "PersonBulkInsertV2"; - new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); - ///////////////////////////////////////////////////////// // start the process - expect to go to the upload step // ///////////////////////////////////////////////////////// - RunProcessInput runProcessInput = new RunProcessInput(); - runProcessInput.setProcessName(processName); - runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); - RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); String processUUID = runProcessOutput.getProcessUUID(); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload"); - ////////////////////////////// - // simulate the file upload // - ////////////////////////////// - String storageReference = UUID.randomUUID() + ".csv"; - StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); - try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) - { - outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); - } - catch(IOException e) - { - throw (e); - } - ////////////////////////// // continue post-upload // ////////////////////////// - runProcessInput.setProcessUUID(processUUID); - runProcessInput.setStartAfterStep("upload"); - runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(2)); assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); @@ -176,29 +148,10 @@ class BulkInsertFullProcessTest extends BaseTest assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); - //////////////////////////////////////////////////////////////////////////////// - // all subsequent steps will want these data - so set up a lambda to set them // - //////////////////////////////////////////////////////////////////////////////// - Consumer addProfileToRunProcessInput = (RunProcessInput input) -> - { - input.addValue("version", "v1"); - input.addValue("layout", "FLAT"); - input.addValue("hasHeaderRow", "true"); - input.addValue("fieldListJSON", JsonUtils.toJson(List.of( - new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), - new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), - new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), - new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), - new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) - ))); - }; - //////////////////////////////// // continue post file-mapping // //////////////////////////////// - runProcessInput.setStartAfterStep("fileMapping"); - addProfileToRunProcessInput.accept(runProcessInput); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostFileMapping(runProcessInput); Serializable valueMappingField = runProcessOutput.getValue("valueMappingField"); assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class); assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName()); @@ -211,23 +164,20 @@ class BulkInsertFullProcessTest extends BaseTest ///////////////////////////////// // continue post value-mapping // ///////////////////////////////// - runProcessInput.setStartAfterStep("valueMapping"); - runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); - addProfileToRunProcessInput.accept(runProcessInput); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostValueMapping(runProcessInput); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); ///////////////////////////////// // continue post review screen // ///////////////////////////////// - runProcessInput.setStartAfterStep("review"); - addProfileToRunProcessInput.accept(runProcessInput); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); assertThat(runProcessOutput.getRecords()).hasSize(2); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); assertThat(runProcessOutput.getException()).isEmpty(); + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2"); + //////////////////////////////////// // query for the inserted records // //////////////////////////////////// @@ -249,4 +199,133 @@ class BulkInsertFullProcessTest extends BaseTest assertNull(records.get(1).getValue("noOfShoes")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneRow() throws Exception + { + /////////////////////////////////////// + // make sure table is empty to start // + /////////////////////////////////////// + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); + + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(1)); + continueProcessPostFileMapping(runProcessInput); + continueProcessPostValueMapping(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // all that just so we can make sure this message is right (because it was wrong when we first wrote it, lol) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id 1"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostReviewScreen(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("review"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostValueMapping(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setStartAfterStep("valueMapping"); + runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); + addProfileToRunProcessInput(runProcessInput); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + private static RunProcessOutput continueProcessPostFileMapping(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("fileMapping"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostUpload(RunProcessInput runProcessInput, String processUUID, StorageInput storageInput) throws QException + { + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("upload"); + runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static StorageInput simulateFileUpload(int noOfRows) throws Exception + { + String storageReference = UUID.randomUUID() + ".csv"; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); + try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) + { + outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + (noOfRows == 2 ? getPersonCsvRow2() : "")).getBytes()); + } + return storageInput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput startProcess(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert"); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addProfileToRunProcessInput(RunProcessInput input) + { + input.addValue("version", "v1"); + input.addValue("layout", "FLAT"); + input.addValue("hasHeaderRow", "true"); + input.addValue("fieldListJSON", JsonUtils.toJson(List.of( + new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), + new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), + new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), + new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), + new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) + ))); + } + } \ No newline at end of file From 2a0bc03337c074a102dffa5d7fe10398df140544 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:14:47 -0600 Subject: [PATCH 56/67] Accept storageReference (file path) as optional input --- .../RenderSavedReportExecuteStep.java | 27 +++++++++++-------- .../RenderSavedReportMetaDataProducer.java | 2 ++ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java index f91a9000..900dc4da 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java @@ -92,16 +92,21 @@ public class RenderSavedReportExecuteStep implements BackendStep //////////////////////////////// // read inputs, set up params // //////////////////////////////// - String sesProviderName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.SES_PROVIDER_NAME); - String fromEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FROM_EMAIL_ADDRESS); - String replyToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.REPLY_TO_EMAIL_ADDRESS); - String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); - ReportFormat reportFormat = ReportFormat.fromString(runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT)); - String sendToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_ADDRESS); - String emailSubject = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_SUBJECT); - SavedReport savedReport = new SavedReport(runBackendStepInput.getRecords().get(0)); - String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, savedReport); - String storageReference = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension(); + String sesProviderName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.SES_PROVIDER_NAME); + String fromEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FROM_EMAIL_ADDRESS); + String replyToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.REPLY_TO_EMAIL_ADDRESS); + String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); + ReportFormat reportFormat = ReportFormat.fromString(runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT)); + String sendToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_ADDRESS); + String emailSubject = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_SUBJECT); + SavedReport savedReport = new SavedReport(runBackendStepInput.getRecords().get(0)); + + String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, savedReport); + String storageReference = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_REFERENCE); + if(!StringUtils.hasContent(storageReference)) + { + storageReference = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension(); + } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if sending an email (or emails), validate the addresses before doing anything so user gets error and can fix // @@ -241,7 +246,7 @@ public class RenderSavedReportExecuteStep implements BackendStep /******************************************************************************* ** *******************************************************************************/ - private String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report) + public static String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmm").withZone(ZoneId.systemDefault()); String datePart = formatter.format(Instant.now()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java index c8693195..4b25c61c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java @@ -56,6 +56,7 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf public static final String FROM_EMAIL_ADDRESS = "fromEmailAddress"; public static final String REPLY_TO_EMAIL_ADDRESS = "replyToEmailAddress"; public static final String FIELD_NAME_STORAGE_TABLE_NAME = "storageTableName"; + public static final String FIELD_NAME_STORAGE_REFERENCE = "storageReference"; public static final String FIELD_NAME_REPORT_FORMAT = "reportFormat"; public static final String FIELD_NAME_EMAIL_ADDRESS = "reportDestinationEmailAddress"; public static final String FIELD_NAME_EMAIL_SUBJECT = "emailSubject"; @@ -81,6 +82,7 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf .withField(new QFieldMetaData(FROM_EMAIL_ADDRESS, QFieldType.STRING)) .withField(new QFieldMetaData(REPLY_TO_EMAIL_ADDRESS, QFieldType.STRING)) .withField(new QFieldMetaData(FIELD_NAME_STORAGE_TABLE_NAME, QFieldType.STRING)) + .withField(new QFieldMetaData(FIELD_NAME_STORAGE_REFERENCE, QFieldType.STRING)) .withRecordListMetaData(new QRecordListMetaData().withTableName(SavedReport.TABLE_NAME))) .withCode(new QCodeReference(RenderSavedReportPreStep.class))) From 92f0bd38467e8310c3bd58afef1a1d6e06d6391a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:15:26 -0600 Subject: [PATCH 57/67] Try to bubble more useful exceptions out --- .../filesystem/sftp/actions/AbstractSFTPAction.java | 13 ++++++++++++- .../filesystem/sftp/utils/SFTPOutputStream.java | 9 +++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 93a5adaf..8d5330ff 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -36,9 +36,11 @@ import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -46,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; @@ -57,6 +60,7 @@ import org.apache.sshd.client.session.ClientSession; import org.apache.sshd.common.config.keys.KeyUtils; import org.apache.sshd.sftp.client.SftpClient; import org.apache.sshd.sftp.client.SftpClientFactory; +import org.apache.sshd.sftp.common.SftpException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -300,9 +304,10 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { + String fullPath = null; try { - String fullPath = getFullBasePath(table, backendBase); + fullPath = getFullBasePath(table, backendBase); if(StringUtils.hasContent(requestedPath)) { fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + requestedPath + File.separatorChar); @@ -340,6 +345,12 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction putFuture; @@ -62,12 +65,14 @@ public class SFTPOutputStream extends PipedOutputStream pipedInputStream = new PipedInputStream(this, 32 * 1024); this.sftpClient = sftpClient; + this.path = path; putFuture = Executors.newSingleThreadExecutor().submit(() -> { try { started.set(true); + LOG.debug("Starting sftp put", logPair("path", path)); sftpClient.put(pipedInputStream, path); } catch(Exception e) @@ -105,6 +110,10 @@ public class SFTPOutputStream extends PipedOutputStream { if(putException.get() != null) { + if(putException.get() instanceof SftpException sftpException) + { + throw new IOException("Error performing SFTP put for path [" + path + "]: " + sftpException.getMessage()); + } throw new IOException("Error performing SFTP put", putException.get()); } From 3ae5f90cc8396512a5f548b8a19eaf180f26f629 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:18:31 -0600 Subject: [PATCH 58/67] Checkstyle --- .../implementations/bulk/insert/BulkInsertFullProcessTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index 91972a91..2fe422fd 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -257,6 +257,9 @@ class BulkInsertFullProcessTest extends BaseTest + /*************************************************************************** + ** + ***************************************************************************/ private static RunProcessOutput continueProcessPostFileMapping(RunProcessInput runProcessInput) throws QException { RunProcessOutput runProcessOutput; From 2808b3fcc4fb7cc24547ba8f457f020b34bd8b54 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:22:49 -0600 Subject: [PATCH 59/67] test fixes --- .../qqq/backend/core/actions/values/QValueFormatterTest.java | 4 ++-- .../qqq/backend/core/scheduler/CronDescriberTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java index c4eac300..05d24862 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java @@ -325,7 +325,7 @@ class QValueFormatterTest extends BaseTest /////////////////////////////////////////////////////////////////////////////////////////// QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); - assertEquals("/data/testTable/47/blobField/Test+Table+47+Blob", record.getValueString("blobField")); + assertEquals("/data/testTable/47/blobField/Test%20Table%2047%20Blob", record.getValueString("blobField")); assertEquals("Test Table 47 Blob", record.getDisplayValue("blobField")); //////////////////////////////////////// @@ -334,7 +334,7 @@ class QValueFormatterTest extends BaseTest adornment.withValue(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION, "html"); record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); - assertEquals("/data/testTable/47/blobField/Test+Table+47+Blob.html", record.getValueString("blobField")); + assertEquals("/data/testTable/47/blobField/Test%20Table%2047%20Blob.html", record.getValueString("blobField")); assertEquals("Test Table 47 Blob.html", record.getDisplayValue("blobField")); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java index b5801d01..35f1e3bf 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java @@ -62,7 +62,7 @@ class CronDescriberTest extends BaseTest assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, from Monday to Friday.", CronDescriber.getDescription("0 0 0 * * MON-FRI")); assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday, Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7")); assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?")); - assertEquals("??", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); + assertEquals("At every 5 seconds starting at 0, 14, 18, 3-39, 52 minutes, every hour, on every day of January, March, September, from Monday to Friday, in 2002-2010.", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); } } \ No newline at end of file From 425d18e6df1dd76f6387fb5b1357b92cf4204357 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 16:03:10 -0600 Subject: [PATCH 60/67] Remove TOOLTIP from FieldAdornment values --- .../javalin/specs/v1/responses/components/FieldAdornment.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java index 2f59ee63..aeebca15 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java @@ -82,6 +82,7 @@ public class FieldAdornment implements ToSchema { EnumSet subSet = EnumSet.allOf(AdornmentType.class); subSet.remove(AdornmentType.FILE_UPLOAD); // todo - remove for next version! + subSet.remove(AdornmentType.TOOLTIP); // todo - remove for next version! FieldAdornmentSubSet.subSet = subSet; } From 7efd8264faafbb364a0725c99b4f93ba67d31181 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 16:56:19 -0600 Subject: [PATCH 61/67] Change tables PVS to be custom type, respecting session permissions; refactor some PVS search logic to make custom implementations a little better --- .../values/QCustomPossibleValueProvider.java | 24 +++ .../SearchPossibleValueSourceAction.java | 140 ++++++++++------ .../TablesCustomPossibleValueProvider.java | 88 ++++++++++ ...esPossibleValueSourceMetaDataProvider.java | 24 +-- ...TablesCustomPossibleValueProviderTest.java | 157 ++++++++++++++++++ 5 files changed, 360 insertions(+), 73 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java index 09e786c5..09bbd74e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; @@ -74,6 +75,29 @@ public interface QCustomPossibleValueProvider } + /*************************************************************************** + * + ***************************************************************************/ + default List> _defaultSearch(SearchPossibleValueSourceInput input, List> possibleValues) + { + SearchPossibleValueSourceAction.PreparedSearchPossibleValueSourceInput preparedInput = SearchPossibleValueSourceAction.prepareSearchPossibleValueSourceInput(input); + + List> rs = new ArrayList<>(); + + for(QPossibleValue possibleValue : possibleValues) + { + if(possibleValue != null && SearchPossibleValueSourceAction.doesPossibleValueMatchSearchInput(possibleValue, preparedInput)) + { + rs.add(possibleValue); + } + } + + rs.sort(Comparator.nullsLast(Comparator.comparing((QPossibleValue pv) -> pv.getLabel()))); + + return (rs); + } + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index 97bcae1c..2063f39f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -51,7 +52,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -108,60 +108,54 @@ public class SearchPossibleValueSourceAction + /*************************************************************************** + ** record to store "computed" values as part of a possible-value search - + ** e.g., ids type-convered, and lower-cased labels. + ***************************************************************************/ + public record PreparedSearchPossibleValueSourceInput(Collection inputIdsAsCorrectType, Collection lowerCaseLabels, String searchTerm) {} + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static PreparedSearchPossibleValueSourceInput prepareSearchPossibleValueSourceInput(SearchPossibleValueSourceInput input) + { + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(input.getPossibleValueSourceName()); + List inputIdsAsCorrectType = convertInputIdsToPossibleValueSourceIdType(possibleValueSource, input.getIdList()); + + Set lowerCaseLabels = null; + if(input.getLabelList() != null) + { + lowerCaseLabels = input.getLabelList().stream() + .filter(Objects::nonNull) + .map(l -> l.toLowerCase()) + .collect(Collectors.toSet()); + } + + return (new PreparedSearchPossibleValueSourceInput(inputIdsAsCorrectType, lowerCaseLabels, input.getSearchTerm())); + } + + + /******************************************************************************* ** *******************************************************************************/ private SearchPossibleValueSourceOutput searchPossibleValueEnum(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) { + PreparedSearchPossibleValueSourceInput preparedSearchPossibleValueSourceInput = prepareSearchPossibleValueSourceInput(input); + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceOutput(); List matchingIds = new ArrayList<>(); - List inputIdsAsCorrectType = convertInputIdsToEnumIdType(possibleValueSource, input.getIdList()); - Set labels = null; - for(QPossibleValue possibleValue : possibleValueSource.getEnumValues()) { - boolean match = false; - - if(input.getIdList() != null) - { - if(inputIdsAsCorrectType.contains(possibleValue.getId())) - { - match = true; - } - } - else if(input.getLabelList() != null) - { - if(labels == null) - { - labels = input.getLabelList().stream().filter(Objects::nonNull).map(l -> l.toLowerCase()).collect(Collectors.toSet()); - } - - if(labels.contains(possibleValue.getLabel().toLowerCase())) - { - match = true; - } - } - else - { - if(StringUtils.hasContent(input.getSearchTerm())) - { - match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.getSearchTerm().toLowerCase()) - || possibleValue.getLabel().toLowerCase().startsWith(input.getSearchTerm().toLowerCase())); - } - else - { - match = true; - } - } + boolean match = doesPossibleValueMatchSearchInput(possibleValue, preparedSearchPossibleValueSourceInput); if(match) { - matchingIds.add((Serializable) possibleValue.getId()); + matchingIds.add(possibleValue.getId()); } - - // todo - skip & limit? - // todo - default filter } List> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, matchingIds); @@ -172,42 +166,84 @@ public class SearchPossibleValueSourceAction + /*************************************************************************** + ** + ***************************************************************************/ + public static boolean doesPossibleValueMatchSearchInput(QPossibleValue possibleValue, PreparedSearchPossibleValueSourceInput input) + { + boolean match = false; + + if(input.inputIdsAsCorrectType() != null) + { + if(input.inputIdsAsCorrectType().contains(possibleValue.getId())) + { + match = true; + } + } + else if(input.lowerCaseLabels() != null) + { + if(input.lowerCaseLabels().contains(possibleValue.getLabel().toLowerCase())) + { + match = true; + } + } + else + { + if(StringUtils.hasContent(input.searchTerm())) + { + match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.searchTerm().toLowerCase()) + || possibleValue.getLabel().toLowerCase().startsWith(input.searchTerm().toLowerCase())); + } + else + { + match = true; + } + } + return match; + } + + + /******************************************************************************* ** The input list of ids might come through as a type that isn't the same as ** the type of the ids in the enum (e.g., strings from a frontend, integers - ** in an enum). So, this method looks at the first id in the enum, and then - ** maps all the inputIds to be of the same type. + ** in an enum). So, this method type-converts them. *******************************************************************************/ - private List convertInputIdsToEnumIdType(QPossibleValueSource possibleValueSource, List inputIdList) + private static List convertInputIdsToPossibleValueSourceIdType(QPossibleValueSource possibleValueSource, List inputIdList) { List rs = new ArrayList<>(); - if(CollectionUtils.nullSafeIsEmpty(inputIdList)) + + if(inputIdList == null) + { + return (null); + } + else if(inputIdList.isEmpty()) { return (rs); } - Object anIdFromTheEnum = possibleValueSource.getEnumValues().get(0).getId(); + QFieldType type = possibleValueSource.getIdType(); for(Serializable inputId : inputIdList) { Object properlyTypedId = null; try { - if(anIdFromTheEnum instanceof Integer) + if(type.equals(QFieldType.INTEGER)) { properlyTypedId = ValueUtils.getValueAsInteger(inputId); } - else if(anIdFromTheEnum instanceof String) + else if(type.isStringLike()) { properlyTypedId = ValueUtils.getValueAsString(inputId); } - else if(anIdFromTheEnum instanceof Boolean) + else if(type.equals(QFieldType.BOOLEAN)) { properlyTypedId = ValueUtils.getValueAsBoolean(inputId); } else { - LOG.warn("Unexpected type [" + anIdFromTheEnum.getClass().getSimpleName() + "] for ids in enum: " + possibleValueSource.getName()); + LOG.warn("Unexpected type [" + type + "] for ids in enum: " + possibleValueSource.getName()); } } catch(Exception e) @@ -215,7 +251,7 @@ public class SearchPossibleValueSourceAction LOG.debug("Error converting possible value id to expected id type", e, logPair("value", inputId)); } - if (properlyTypedId != null) + if(properlyTypedId != null) { rs.add(properlyTypedId); } @@ -397,7 +433,7 @@ public class SearchPossibleValueSourceAction } catch(Exception e) { - String message = "Error sending searching custom possible value source [" + input.getPossibleValueSourceName() + "]"; + String message = "Error searching custom possible value source [" + input.getPossibleValueSourceName() + "]"; LOG.warn(message, e); throw (new QException(message)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java new file mode 100644 index 00000000..93e97d83 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue)); + if(table != null && !table.getIsHidden()) + { + PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table); + if(PermissionCheckResult.ALLOW.equals(permissionCheckResult)) + { + return (new QPossibleValue<>(table.getName(), table.getLabel())); + } + } + + return null; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////// + // build all of the possible values (note, will be filtered by user's permissions) // + ///////////////////////////////////////////////////////////////////////////////////// + List> allPossibleValues = new ArrayList<>(); + for(QTableMetaData table : QContext.getQInstance().getTables().values()) + { + QPossibleValue possibleValue = getPossibleValue(table.getName()); + if(possibleValue != null) + { + allPossibleValues.add(possibleValue); + } + } + + return _defaultSearch(input, allPossibleValues); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java index 27cce6df..75087ae7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java @@ -22,17 +22,11 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; -import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import org.apache.commons.lang.BooleanUtils; /******************************************************************************* @@ -51,22 +45,10 @@ public class TablesPossibleValueSourceMetaDataProvider { QPossibleValueSource possibleValueSource = new QPossibleValueSource() .withName(NAME) - .withType(QPossibleValueSourceType.ENUM) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class)) .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY); - List> enumValues = new ArrayList<>(); - for(QTableMetaData table : qInstance.getTables().values()) - { - if(BooleanUtils.isNotTrue(table.getIsHidden())) - { - String label = StringUtils.hasContent(table.getLabel()) ? table.getLabel() : QInstanceEnricher.nameToLabel(table.getName()); - enumValues.add(new QPossibleValue<>(table.getName(), label)); - } - } - - enumValues.sort(Comparator.comparing(QPossibleValue::getLabel)); - - possibleValueSource.withEnumValues(enumValues); return (possibleValueSource); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java new file mode 100644 index 00000000..4bdbbc06 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for TablesCustomPossibleValueProvider + *******************************************************************************/ +class TablesCustomPossibleValueProviderTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() + { + QInstance qInstance = TestUtils.defineInstance(); + + qInstance.addTable(new QTableMetaData() + .withName("hidden") + .withIsHidden(true) + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER))); + + qInstance.addTable(new QTableMetaData() + .withName("restricted") + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)) + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER))); + + qInstance.addPossibleValueSource(TablesPossibleValueSourceMetaDataProvider.defineTablesPossibleValueSource(qInstance)); + + QContext.init(qInstance, newSession()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetPossibleValue() + { + TablesCustomPossibleValueProvider provider = new TablesCustomPossibleValueProvider(); + + QPossibleValue possibleValue = provider.getPossibleValue(TestUtils.TABLE_NAME_PERSON); + assertEquals(TestUtils.TABLE_NAME_PERSON, possibleValue.getId()); + assertEquals("Person", possibleValue.getLabel()); + + assertNull(provider.getPossibleValue("no-such-table")); + assertNull(provider.getPossibleValue("hidden")); + assertNull(provider.getPossibleValue("restricted")); + + QContext.getQSession().withPermission("restricted.hasAccess"); + assertNotNull(provider.getPossibleValue("restricted")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPossibleValue() throws QException + { + TablesCustomPossibleValueProvider provider = new TablesCustomPossibleValueProvider(); + + List> list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).noneMatch(p -> p.getId().equals("no-such-table")); + assertThat(list).noneMatch(p -> p.getId().equals("hidden")); + assertThat(list).noneMatch(p -> p.getId().equals("restricted")); + assertNull(provider.getPossibleValue("restricted")); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withIdList(List.of(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_SHAPE, "hidden"))); + assertEquals(2, list.size()); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE)); + assertThat(list).noneMatch(p -> p.getId().equals("hidden")); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withLabelList(List.of("Person", "Shape", "Restricted"))); + assertEquals(2, list.size()); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE)); + assertThat(list).noneMatch(p -> p.getId().equals("restricted")); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withSearchTerm("restricted")); + assertEquals(0, list.size()); + + ///////////////////////////////////////// + // add permission for restricted table // + ///////////////////////////////////////// + QContext.getQSession().withPermission("restricted.hasAccess"); + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withSearchTerm("restricted")); + assertEquals(1, list.size()); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withLabelList(List.of("Person", "Shape", "Restricted"))); + assertEquals(3, list.size()); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE)); + assertThat(list).anyMatch(p -> p.getId().equals("restricted")); + + } + +} \ No newline at end of file From 9fb53af0ba71d5599aa508d7c3f478d04ea89690 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 17:01:40 -0600 Subject: [PATCH 62/67] Checkstyle! also rename new method --- .../core/actions/values/QCustomPossibleValueProvider.java | 6 ++++-- .../metadata/tables/TablesCustomPossibleValueProvider.java | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java index 09bbd74e..d32e4e79 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java @@ -76,9 +76,11 @@ public interface QCustomPossibleValueProvider /*************************************************************************** - * + ** meant to be protected (but interface...) - for a custom PVS implementation + ** to complete its search (e.g., after it generates the list of PVS objects, + ** let this method do the filtering). ***************************************************************************/ - default List> _defaultSearch(SearchPossibleValueSourceInput input, List> possibleValues) + default List> completeCustomPVSSearch(SearchPossibleValueSourceInput input, List> possibleValues) { SearchPossibleValueSourceAction.PreparedSearchPossibleValueSourceInput preparedInput = SearchPossibleValueSourceAction.prepareSearchPossibleValueSourceInput(input); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java index 93e97d83..df2b55a3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java @@ -82,7 +82,7 @@ public class TablesCustomPossibleValueProvider implements QCustomPossibleValuePr } } - return _defaultSearch(input, allPossibleValues); + return completeCustomPVSSearch(input, allPossibleValues); } } From 99e282fcdfcf6421f880179dbeccc9a2b796e572 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 28 Feb 2025 19:41:57 -0600 Subject: [PATCH 63/67] Add sourceClass attribute to MetaDataProducerInterface --- .../core/model/metadata/MetaDataProducer.java | 35 ++++++++++ .../metadata/MetaDataProducerHelper.java | 65 +++++++++++-------- .../metadata/MetaDataProducerInterface.java | 19 ++++++ ...omRecordEntityGenericMetaDataProducer.java | 34 ++++++++++ ...omRecordEntityGenericMetaDataProducer.java | 34 ++++++++++ ...ueSourceOfEnumGenericMetaDataProducer.java | 37 +++++++++++ ...eSourceOfTableGenericMetaDataProducer.java | 35 ++++++++++ ...dEntityToTableGenericMetaDataProducer.java | 34 ++++++++++ 8 files changed, 266 insertions(+), 27 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java index a4dbe375..401ecb0e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java @@ -29,5 +29,40 @@ package com.kingsrook.qqq.backend.core.model.metadata; *******************************************************************************/ public abstract class MetaDataProducer implements MetaDataProducerInterface { + private Class sourceClass; + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + @Override + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + @Override + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public MetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index 487c6dda..ec41e0b6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -239,17 +239,19 @@ public class MetaDataProducerHelper ** ***************************************************************************/ @SuppressWarnings("unchecked") - private static > MetaDataProducerInterface processMetaDataProducingPossibleValueEnum(Class aClass) + private static > MetaDataProducerInterface processMetaDataProducingPossibleValueEnum(Class sourceClass) { String warningPrefix = "Found a class annotated as @" + QMetaDataProducingPossibleValueEnum.class.getSimpleName(); - if(!PossibleValueEnum.class.isAssignableFrom(aClass)) + if(!PossibleValueEnum.class.isAssignableFrom(sourceClass)) { - LOG.warn(warningPrefix + ", but which is not a " + PossibleValueEnum.class.getSimpleName() + ", so it will not be used.", logPair("class", aClass.getSimpleName())); + LOG.warn(warningPrefix + ", but which is not a " + PossibleValueEnum.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName())); return null; } - PossibleValueEnum[] values = (PossibleValueEnum[]) aClass.getEnumConstants(); - return (new PossibleValueSourceOfEnumGenericMetaDataProducer(aClass.getSimpleName(), (PossibleValueEnum[]) values)); + PossibleValueEnum[] values = (PossibleValueEnum[]) sourceClass.getEnumConstants(); + PossibleValueSourceOfEnumGenericMetaDataProducer producer = new PossibleValueSourceOfEnumGenericMetaDataProducer<>(sourceClass.getSimpleName(), (PossibleValueEnum[]) values); + producer.setSourceClass(sourceClass); + return producer; } @@ -257,32 +259,32 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static List> processMetaDataProducingEntity(Class aClass) throws Exception + private static List> processMetaDataProducingEntity(Class sourceClass) throws Exception { List> rs = new ArrayList<>(); - QMetaDataProducingEntity qMetaDataProducingEntity = aClass.getAnnotation(QMetaDataProducingEntity.class); + QMetaDataProducingEntity qMetaDataProducingEntity = sourceClass.getAnnotation(QMetaDataProducingEntity.class); String warningPrefix = "Found a class annotated as @" + QMetaDataProducingEntity.class.getSimpleName(); /////////////////////////////////////////////////////////// // make sures class is QRecordEntity and cast it as such // /////////////////////////////////////////////////////////// - if(!QRecordEntity.class.isAssignableFrom(aClass)) + if(!QRecordEntity.class.isAssignableFrom(sourceClass)) { - LOG.warn(warningPrefix + ", but which is not a " + QRecordEntity.class.getSimpleName() + ", so it will not be used.", logPair("class", aClass.getSimpleName())); + LOG.warn(warningPrefix + ", but which is not a " + QRecordEntity.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName())); return (rs); } @SuppressWarnings("unchecked") // safe per the check above. - Class recordEntityClass = (Class) aClass; + Class recordEntityClass = (Class) sourceClass; //////////////////////////////////////////////// // get TABLE_NAME static field from the class // //////////////////////////////////////////////// - Field tableNameField = aClass.getDeclaredField("TABLE_NAME"); + Field tableNameField = recordEntityClass.getDeclaredField("TABLE_NAME"); if(!tableNameField.getType().equals(String.class)) { - LOG.warn(warningPrefix + ", but whose TABLE_NAME field is not a String, so it will not be used.", logPair("class", aClass.getSimpleName())); + LOG.warn(warningPrefix + ", but whose TABLE_NAME field is not a String, so it will not be used.", logPair("class", recordEntityClass.getSimpleName())); return (rs); } @@ -303,6 +305,7 @@ public class MetaDataProducerHelper } RecordEntityToTableGenericMetaDataProducer producer = new RecordEntityToTableGenericMetaDataProducer(tableNameValue, recordEntityClass, tableMetaDataProductionCustomizer); + producer.setSourceClass(recordEntityClass); if(tableMetaDataCustomizer != null) { @@ -322,7 +325,9 @@ public class MetaDataProducerHelper //////////////////////////////////////// if(qMetaDataProducingEntity.producePossibleValueSource()) { - rs.add(new PossibleValueSourceOfTableGenericMetaDataProducer(tableNameValue)); + PossibleValueSourceOfTableGenericMetaDataProducer producer = new PossibleValueSourceOfTableGenericMetaDataProducer(tableNameValue); + producer.setSourceClass(recordEntityClass); + rs.add(producer); } ////////////////////////// @@ -333,11 +338,11 @@ public class MetaDataProducerHelper Class childEntityClass = childTable.childTableEntityClass(); if(childTable.childJoin().enabled()) { - CollectionUtils.addIfNotNull(rs, processChildJoin(aClass, childTable)); + CollectionUtils.addIfNotNull(rs, processChildJoin(recordEntityClass, childTable)); if(childTable.childRecordListWidget().enabled()) { - CollectionUtils.addIfNotNull(rs, processChildRecordListWidget(aClass, childTable)); + CollectionUtils.addIfNotNull(rs, processChildRecordListWidget(recordEntityClass, childTable)); } } else @@ -347,7 +352,7 @@ public class MetaDataProducerHelper ////////////////////////////////////////////////////////////////////////// // if not doing the join, can't do the child-widget, so warn about that // ////////////////////////////////////////////////////////////////////////// - LOG.warn(warningPrefix + " requested to produce a ChildRecordListWidget, but not produce a Join - which is not allowed (must do join to do widget). ", logPair("class", aClass.getSimpleName()), logPair("childEntityClass", childEntityClass.getSimpleName())); + LOG.warn(warningPrefix + " requested to produce a ChildRecordListWidget, but not produce a Join - which is not allowed (must do join to do widget). ", logPair("class", recordEntityClass.getSimpleName()), logPair("childEntityClass", childEntityClass.getSimpleName())); } } } @@ -360,14 +365,16 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static MetaDataProducerInterface processChildRecordListWidget(Class aClass, ChildTable childTable) throws Exception + private static MetaDataProducerInterface processChildRecordListWidget(Class sourceClass, ChildTable childTable) throws Exception { Class childEntityClass = childTable.childTableEntityClass(); - String parentTableName = getTableNameStaticFieldValue(aClass); + String parentTableName = getTableNameStaticFieldValue(sourceClass); String childTableName = getTableNameStaticFieldValue(childEntityClass); ChildRecordListWidget childRecordListWidget = childTable.childRecordListWidget(); - return (new ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, childRecordListWidget)); + ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer producer = new ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, childRecordListWidget); + producer.setSourceClass(sourceClass); + return producer; } @@ -397,20 +404,22 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static MetaDataProducerInterface processChildJoin(Class aClass, ChildTable childTable) throws Exception + private static MetaDataProducerInterface processChildJoin(Class entityClass, ChildTable childTable) throws Exception { Class childEntityClass = childTable.childTableEntityClass(); - String parentTableName = getTableNameStaticFieldValue(aClass); + String parentTableName = getTableNameStaticFieldValue(entityClass); String childTableName = getTableNameStaticFieldValue(childEntityClass); String possibleValueFieldName = findPossibleValueField(childEntityClass, parentTableName); if(!StringUtils.hasContent(possibleValueFieldName)) { - LOG.warn("Could not find field in [" + childEntityClass.getSimpleName() + "] with possibleValueSource referencing table [" + aClass.getSimpleName() + "]"); + LOG.warn("Could not find field in [" + childEntityClass.getSimpleName() + "] with possibleValueSource referencing table [" + entityClass.getSimpleName() + "]"); return (null); } - return (new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName)); + ChildJoinFromRecordEntityGenericMetaDataProducer producer = new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName); + producer.setSourceClass(entityClass); + return producer; } @@ -418,18 +427,20 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static MetaDataProducerInterface processMetaDataProducer(Class aClass) throws Exception + private static MetaDataProducerInterface processMetaDataProducer(Class sourceCClass) throws Exception { - for(Constructor constructor : aClass.getConstructors()) + for(Constructor constructor : sourceCClass.getConstructors()) { if(constructor.getParameterCount() == 0) { Object o = constructor.newInstance(); - return (MetaDataProducerInterface) o; + MetaDataProducerInterface producer = (MetaDataProducerInterface) o; + producer.setSourceClass(sourceCClass); + return producer; } } - LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", aClass.getSimpleName())); + LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", sourceCClass.getSimpleName())); return null; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java index fa725451..6aef051e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java @@ -73,4 +73,23 @@ public interface MetaDataProducerInterface return (true); } + + /*************************************************************************** + * + ***************************************************************************/ + default void setSourceClass(Class sourceClass) + { + ////////// + // noop // + ////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default Class getSourceClass() + { + return null; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java index 76c892cd..c1bc51cf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java @@ -62,6 +62,7 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat private String parentTableName; // e.g., order private String foreignKeyFieldName; // e.g., orderId + private Class sourceClass; /*************************************************************************** @@ -102,4 +103,37 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat return (join); } + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public ChildJoinFromRecordEntityGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java index d9d7858c..6dac1a10 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java @@ -57,6 +57,8 @@ public class ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer implem private ChildRecordListWidget childRecordListWidget; + private Class sourceClass; + /*************************************************************************** @@ -111,4 +113,36 @@ public class ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer implem return (widget); } + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java index 8cb96ec2..f43818b5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java @@ -40,6 +40,10 @@ public class PossibleValueSourceOfEnumGenericMetaDataProducer[] values; + private Class sourceClass; + + + /******************************************************************************* @@ -62,4 +66,37 @@ public class PossibleValueSourceOfEnumGenericMetaDataProducer getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public PossibleValueSourceOfEnumGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java index c1656f35..7d194f64 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java @@ -37,6 +37,7 @@ public class PossibleValueSourceOfTableGenericMetaDataProducer implements MetaDa { private final String tableName; + private Class sourceClass; /******************************************************************************* @@ -58,4 +59,38 @@ public class PossibleValueSourceOfTableGenericMetaDataProducer implements MetaDa { return (QPossibleValueSource.newForTable(tableName)); } + + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public PossibleValueSourceOfTableGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java index 21565148..20cf5be1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java @@ -48,6 +48,7 @@ public class RecordEntityToTableGenericMetaDataProducer implements MetaDataProdu private static MetaDataCustomizerInterface defaultMetaDataCustomizer = null; + private Class sourceClass; /******************************************************************************* @@ -154,4 +155,37 @@ public class RecordEntityToTableGenericMetaDataProducer implements MetaDataProdu RecordEntityToTableGenericMetaDataProducer.defaultMetaDataCustomizer = defaultMetaDataCustomizer; } + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public RecordEntityToTableGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } From 4b0d093a4a06f35e42baff4a0858b8523ca26be2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 28 Feb 2025 19:42:40 -0600 Subject: [PATCH 64/67] Add clearKey(key) --- .../backend/core/utils/memoization/Memoization.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java index 4ed6cbce..ac02e21b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java @@ -247,6 +247,16 @@ public class Memoization + /******************************************************************************* + ** + *******************************************************************************/ + public void clearKey(K key) + { + this.map.remove(key); + } + + + /******************************************************************************* ** Setter for timeoutSeconds ** From 4cbcd0a1497c781efc0b45b8e7b807ebbfe4f21c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 28 Feb 2025 19:45:01 -0600 Subject: [PATCH 65/67] better handling of some - ranges; upper-case input string to match month/day names; handle '*' day of week; day-names in , case; hour w/ AM/PM in , case; join with commas and and. --- .../backend/core/scheduler/CronDescriber.java | 41 +++++++++---------- .../core/scheduler/CronDescriberTest.java | 19 ++++++--- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java index 2b2a2c56..40099988 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -94,7 +95,7 @@ public class CronDescriber ***************************************************************************/ public static String getDescription(String cronExpression) throws ParseException { - String[] parts = cronExpression.split("\\s+"); + String[] parts = cronExpression.trim().toUpperCase().split("\\s+"); if(parts.length < 6 || parts.length > 7) { throw new ParseException("Invalid cron expression: " + cronExpression, 0); @@ -173,15 +174,16 @@ public class CronDescriber { return "every month"; } + else if(month.contains("-")) + { + String[] parts = month.split("-"); + return String.format("%s to %s", MONTH_MAP.getOrDefault(parts[0], parts[0]), MONTH_MAP.getOrDefault(parts[1], parts[1])); + } else { - String[] months = month.split(","); - StringBuilder result = new StringBuilder(); - for(String m : months) - { - result.append(MONTH_MAP.getOrDefault(m, m)).append(", "); - } - return result.substring(0, result.length() - 2); + String[] months = month.split(","); + List monthNames = Arrays.stream(months).map(m -> MONTH_MAP.getOrDefault(m, m)).toList(); + return StringUtils.joinWithCommasAndAnd(monthNames); } } @@ -192,7 +194,7 @@ public class CronDescriber ***************************************************************************/ private static String describeDayOfWeek(String dayOfWeek) { - if(dayOfWeek.equals("?")) + if(dayOfWeek.equals("?") || dayOfWeek.equals("*")) { return "every day of the week"; } @@ -212,13 +214,9 @@ public class CronDescriber } else { - String[] days = dayOfWeek.split(","); - StringBuilder result = new StringBuilder(); - for(String d : days) - { - result.append(DAY_OF_WEEK_MAP.getOrDefault(d, d)).append(", "); - } - return result.substring(0, result.length() - 2); + String[] days = dayOfWeek.split(","); + List dayNames = Arrays.stream(days).map(d -> DAY_OF_WEEK_MAP.getOrDefault(d, d)).toList(); + return StringUtils.joinWithCommasAndAnd(dayNames); } } @@ -244,21 +242,22 @@ public class CronDescriber } else if(part.contains(",")) { + List partsList = Arrays.stream(part.split(",")).toList(); + if(label.equals("hour")) { - String[] parts = part.split(","); - List partList = Arrays.stream(parts).map(p -> hourToAmPm(p)).toList(); - return String.join(", ", partList); + List hourNames = partsList.stream().map(p -> hourToAmPm(p)).toList(); + return StringUtils.joinWithCommasAndAnd(hourNames); } else { if(label.equals("day")) { - return "days " + part.replace(",", ", "); + return "days " + StringUtils.joinWithCommasAndAnd(partsList); } else { - return part.replace(",", ", ") + " " + label + "s"; + return StringUtils.joinWithCommasAndAnd(partsList) + " " + label + "s"; } } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java index 35f1e3bf..3b259ce9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java @@ -43,7 +43,7 @@ class CronDescriberTest extends BaseTest assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * * * ?")); assertEquals("At 0 seconds, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 * * * * ?")); assertEquals("At 0 seconds, 0 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 * * * ?")); - assertEquals("At 0 seconds, 0, 30 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0,30 * * * ?")); + assertEquals("At 0 seconds, 0 and 30 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0,30 * * * ?")); assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 0 * * ?")); assertEquals("At 0 seconds, 0 minutes, 1 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 1 * * ?")); assertEquals("At 0 seconds, 0 minutes, 11 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 11 * * ?")); @@ -51,18 +51,25 @@ class CronDescriberTest extends BaseTest assertEquals("At 0 seconds, 0 minutes, 1 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 13 * * ?")); assertEquals("At 0 seconds, 0 minutes, 11 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 23 * * ?")); assertEquals("At 0 seconds, 0 minutes, midnight, on day 10 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10 * ?")); - assertEquals("At 0 seconds, 0 minutes, midnight, on days 10, 20 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10,20 * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on days 10 and 20 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10,20 * ?")); assertEquals("At 0 seconds, 0 minutes, midnight, on days from 10 to 15 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10-15 * ?")); assertEquals("At from 10 to 15 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("10-15 0 0 * * ?")); assertEquals("At 30 seconds, 30 minutes, from 8 AM to 4 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("30 30 8-16 * * ?")); assertEquals("At 0 seconds, 0 minutes, midnight, on every 3 days starting at 0 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 */3 * ?")); assertEquals("At every 5 seconds starting at 0, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0/5 0 0 * * ?")); assertEquals("At 0 seconds, every 30 minutes starting at 3, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 3/30 0 * * ?")); - assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Monday, Wednesday, Friday.", CronDescriber.getDescription("0 0 0 * * MON,WED,FRI")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Monday, Wednesday, and Friday.", CronDescriber.getDescription("0 0 0 * * MON,WED,FRI")); assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, from Monday to Friday.", CronDescriber.getDescription("0 0 0 * * MON-FRI")); - assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday, Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7")); - assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?")); - assertEquals("At every 5 seconds starting at 0, 14, 18, 3-39, 52 minutes, every hour, on every day of January, March, September, from Monday to Friday, in 2002-2010.", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday and Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7")); + assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, and 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?")); + assertEquals("At every 5 seconds starting at 0, 14, 18, 3-39, and 52 minutes, every hour, on every day of January, March, and September, from Monday to Friday, in 2002-2010.", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); + + assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * ? * *")); + assertEquals("At every second, every minute, every hour, on every day of January to June, every day of the week.", CronDescriber.getDescription("* * * ? 1-6 *")); + assertEquals("At every second, every minute, every hour, on days 1, 3, and 5 of every month, every day of the week.", CronDescriber.getDescription("* * * 1,3,5 * *")); + // todo fix has 2-4 hours and 3 PM, s/b 2 AM to 4 AM and 3 PM assertEquals("At every second, every minute, every hour, on days 1, 3, and 5 of every month, every day of the week.", CronDescriber.getDescription("* * 2-4,15 1,3,5 * *")); + // hour failing on 3,2-7 (at least in TS side?) + // 3,2-7 makes 3,2 to July } } \ No newline at end of file From d4d20e2b2016f40f99c0f33ecaf751dd43b9891f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 28 Feb 2025 19:53:18 -0600 Subject: [PATCH 66/67] Fix this test that would never have worked on 3/1 of a non-leap year, i suppose --- .../actions/tables/query/expressions/NowWithOffsetTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java index 4655255d..affa9325 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java @@ -66,7 +66,7 @@ class NowWithOffsetTest extends BaseTest assertThat(twoWeeksFromNowMillis).isCloseTo(now + (14 * DAY_IN_MILLIS), allowedDiff); long oneMonthAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli(); - assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusOneDay); + assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusTwoDays); // two days, to work on 3/1... long twoMonthsFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli(); assertThat(twoMonthsFromNowMillis).isCloseTo(now + (60 * DAY_IN_MILLIS), allowedDiffPlusTwoDays); From 3a8bfe5f486e02ffd8706748e201cdeac9028d75 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 3 Mar 2025 07:49:57 -0600 Subject: [PATCH 67/67] Minor cleanup from code review (comments, fixed a few exceptions); --- .../qqq/backend/core/actions/tables/InsertAction.java | 2 +- .../core/actions/values/SearchPossibleValueSourceAction.java | 2 +- .../qqq/backend/core/instances/QInstanceEnricher.java | 3 ++- .../metadata/tables/TablesCustomPossibleValueProvider.java | 3 ++- qqq-backend-module-filesystem/pom.xml | 1 + 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 846ed476..5ce2ce38 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -114,7 +114,7 @@ public class InsertAction extends AbstractQActionFunction { diff --git a/qqq-backend-module-filesystem/pom.xml b/qqq-backend-module-filesystem/pom.xml index 9344db4a..c389a018 100644 --- a/qqq-backend-module-filesystem/pom.xml +++ b/qqq-backend-module-filesystem/pom.xml @@ -75,6 +75,7 @@ test + net.java.dev.jna jna 5.7.0