diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index a535ae47..806b0f5c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -815,13 +815,14 @@ public class RunProcessAction { QSession session = QContext.getQSession(); QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend()); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) { LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"); } else { - basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue()); + basepullKeyValue += "-" + session.getBackendVariants().get(variantTypeKey); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index 0ded3c93..a5028027 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -82,6 +82,11 @@ public class DeleteAction { ActionHelper.validateSession(deleteInput); + if(deleteInput.getTableName() == null) + { + throw (new QException("Table name was not specified in delete input")); + } + QTableMetaData table = deleteInput.getTable(); String primaryKeyFieldName = table.getPrimaryKeyField(); QFieldMetaData primaryKeyField = table.getField(primaryKeyFieldName); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 64a4ddc3..846ed476 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -110,6 +111,12 @@ public class InsertAction extends AbstractQActionFunction fileNameFormatFields = (List) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS); - List values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList(); + List values = CollectionUtils.nullSafeHasContents(fileNameFormatFields) ? fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList() : Collections.emptyList(); fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values); } } @@ -531,7 +535,7 @@ public class QValueFormatter //////////////////////////////////////////////////////////////////////////////////////////////// // if field type is blob OR if there's a supplemental process or code-ref that needs to run - // - // then update its value to be a callback-url that'll give access to the bytes to download // + // then update its value to be a callback-url that'll give access to the bytes to download. // // implied here is that a String value (w/o supplemental code/proc) has its value stay as a // // URL, which is where the file is directly downloaded from. And in the case of a String // // with code-to-run, then the code should run, followed by a redirect to the value URL. // @@ -540,7 +544,10 @@ public class QValueFormatter || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE) || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME)) { - record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName); + record.setValue(field.getName(), "/data/" + table.getName() + "/" + + URLEncoder.encode(ValueUtils.getValueAsString(primaryKey), StandardCharsets.UTF_8) + "/" + + field.getName() + "/" + + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8)); } record.setDisplayValue(field.getName(), fileName); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index dcc06391..a214ee6b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -37,7 +37,9 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TimeZone; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -108,12 +110,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.Automatio import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; +import org.apache.commons.lang.BooleanUtils; import org.quartz.CronExpression; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -543,6 +549,60 @@ public class QInstanceValidator { assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + "."); + /////////////////////// + // validate variants // + /////////////////////// + BackendVariantsConfig backendVariantsConfig = backend.getBackendVariantsConfig(); + if(BooleanUtils.isTrue(backend.getUsesVariants())) + { + if(assertCondition(backendVariantsConfig != null, "Missing backendVariantsConfig in backend [" + backendName + "] which is marked as usesVariants")) + { + assertCondition(StringUtils.hasContent(backendVariantsConfig.getVariantTypeKey()), "Missing variantTypeKey in backendVariantsConfig in [" + backendName + "]"); + + String optionsTableName = backendVariantsConfig.getOptionsTableName(); + QTableMetaData optionsTable = qInstance.getTable(optionsTableName); + if(assertCondition(StringUtils.hasContent(optionsTableName), "Missing optionsTableName in backendVariantsConfig in [" + backendName + "]")) + { + if(assertCondition(optionsTable != null, "Unrecognized optionsTableName [" + optionsTableName + "] in backendVariantsConfig in [" + backendName + "]")) + { + QQueryFilter optionsFilter = backendVariantsConfig.getOptionsFilter(); + if(optionsFilter != null) + { + validateQueryFilter(qInstance, "optionsFilter in backendVariantsConfig in backend [" + backendName + "]: ", optionsTable, optionsFilter, null); + } + } + } + + Map backendSettingSourceFieldNameMap = backendVariantsConfig.getBackendSettingSourceFieldNameMap(); + if(assertCondition(CollectionUtils.nullSafeHasContents(backendSettingSourceFieldNameMap), "Missing or empty backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]")) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // only validate field names in the backendSettingSourceFieldNameMap if there is NOT a variantRecordSupplier // + // (the idea being, that the supplier might be building a record with fieldNames that aren't in the table... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(optionsTable != null && backendVariantsConfig.getVariantRecordLookupFunction() == null) + { + for(Map.Entry entry : backendSettingSourceFieldNameMap.entrySet()) + { + assertCondition(optionsTable.getFields().containsKey(entry.getValue()), "Unrecognized fieldName [" + entry.getValue() + "] in backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]"); + } + } + } + + if(backendVariantsConfig.getVariantRecordLookupFunction() != null) + { + validateSimpleCodeReference("VariantRecordSupplier in backendVariantsConfig in backend [" + backendName + "]: ", backendVariantsConfig.getVariantRecordLookupFunction(), UnsafeFunction.class, Function.class); + } + } + } + else + { + assertCondition(backendVariantsConfig == null, "Should not have a backendVariantsConfig in backend [" + backendName + "] which is not marked as usesVariants"); + } + + /////////////////////////////////////////// + // let the backend do its own validation // + /////////////////////////////////////////// backend.performValidation(this); runPlugins(QBackendMetaData.class, backend, qInstance); @@ -1356,7 +1416,7 @@ public class QInstanceValidator //////////////////////////////////////////////////////////////////////// if(customizerInstance != null && tableCustomizer.getExpectedType() != null) { - assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance); + assertObjectCanBeCasted(prefix, customizerInstance, tableCustomizer.getExpectedType()); } } } @@ -1368,18 +1428,31 @@ public class QInstanceValidator /******************************************************************************* ** Make sure that a given object can be casted to an expected type. *******************************************************************************/ - private T assertObjectCanBeCasted(String errorPrefix, Class expectedType, Object object) + private void assertObjectCanBeCasted(String errorPrefix, Object object, Class... anyOfExpectedClasses) { - T castedObject = null; - try + for(Class expectedClass : anyOfExpectedClasses) { - castedObject = expectedType.cast(object); + try + { + expectedClass.cast(object); + return; + } + catch(ClassCastException e) + { + ///////////////////////////////////// + // try next type (if there is one) // + ///////////////////////////////////// + } } - catch(ClassCastException e) + + if(anyOfExpectedClasses.length == 1) { - errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType); + errors.add(errorPrefix + "CodeReference is not of the expected type: " + anyOfExpectedClasses[0]); + } + else + { + errors.add(errorPrefix + "CodeReference is not any of the expected types: " + Arrays.stream(anyOfExpectedClasses).map(c -> c.getName()).collect(Collectors.joining(", "))); } - return castedObject; } @@ -1616,12 +1689,12 @@ public class QInstanceValidator for(QFieldMetaData field : process.getInputFields()) { - validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName()); + validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName() + " "); } for(QFieldMetaData field : process.getOutputFields()) { - validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName()); + validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName() + " "); } if(process.getCancelStep() != null) @@ -2123,7 +2196,8 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class expectedClass) + @SafeVarargs + private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class... anyOfExpectedClasses) { if(!preAssertionsForCodeReference(codeReference, prefix)) { @@ -2151,7 +2225,7 @@ public class QInstanceValidator //////////////////////////////////////////////////////////////////////// if(classInstance != null) { - assertObjectCanBeCasted(prefix, expectedClass, classInstance); + assertObjectCanBeCasted(prefix, classInstance, anyOfExpectedClasses); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java index 83b8785c..35b09c45 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java @@ -326,6 +326,20 @@ public class AuditSingleInput implements Serializable + /******************************************************************************* + ** Fluent setter for details + *******************************************************************************/ + public AuditSingleInput withDetailMessages(List details) + { + for(String detail : details) + { + addDetail(message); + } + return (this); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 17b3e608..84da02f7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes; import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -34,9 +35,11 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface; @@ -247,6 +250,26 @@ public class RunBackendStepInput extends AbstractActionInput + /******************************************************************************* + ** Getter for records converted to entities of a given type. + ** + *******************************************************************************/ + public List getRecordsAsEntities(Class entityClass) throws QException + { + List rs = new ArrayList<>(); + + /////////////////////////////////////////////////////////////////////////////////// + // note - important to call getRecords here, which is overwritten in subclasses! // + /////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : getRecords()) + { + rs.add(QRecordEntity.fromQRecord(entityClass, record)); + } + return (rs); + } + + + /******************************************************************************* ** Setter for records ** @@ -582,7 +605,7 @@ public class RunBackendStepInput extends AbstractActionInput ***************************************************************************/ public void traceMessage(ProcessTracerMessage message) { - if(processTracer != null) + if(processTracer != null && message != null) { try { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java index 7c89b98a..ba6b87c5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java @@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -258,7 +259,7 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial /******************************************************************************* - ** + ** add a record to the step output, e.g., for going through to the next step. *******************************************************************************/ public void addRecord(QRecord record) { @@ -271,6 +272,16 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial + /*************************************************************************** + ** add a RecordEntity to the step output, e.g., for going through to the next step. + ***************************************************************************/ + public void addRecordEntity(QRecordEntity recordEntity) + { + addRecord(recordEntity.toQRecord()); + } + + + /******************************************************************************* ** Getter for auditInputList *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java new file mode 100644 index 00000000..04c4e4eb --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java @@ -0,0 +1,47 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import com.kingsrook.qqq.backend.core.logging.QLogger; + + +/******************************************************************************* + ** for use-cases where a metaDataProducer directly adds its objects to the + ** qInstance, then this empty object can be returned. + *******************************************************************************/ +public class EmptyMetaDataProducerOutput implements MetaDataProducerOutput +{ + private static final QLogger LOG = QLogger.getLogger(EmptyMetaDataProducerOutput.class); + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void addSelfToInstance(QInstance instance) + { + ///////////////////////////////// + // noop - this output is empty // + ///////////////////////////////// + LOG.trace("empty meta data producer has nothing to add."); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index 52f6c95f..9bc7ad04 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -26,8 +26,13 @@ import java.util.HashSet; import java.util.Set; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.serialization.QBackendMetaDataDeserializer; import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; +import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -45,21 +50,18 @@ public class QBackendMetaData implements TopLevelMetaDataInterface private Set enabledCapabilities = new HashSet<>(); private Set disabledCapabilities = new HashSet<>(); - private Boolean usesVariants = false; - private String variantOptionsTableIdField; - private String variantOptionsTableNameField; - private String variantOptionsTableTypeField; - private String variantOptionsTableTypeValue; - private String variantOptionsTableUsernameField; - private String variantOptionsTablePasswordField; - private String variantOptionsTableApiKeyField; - private String variantOptionsTableClientIdField; - private String variantOptionsTableClientSecretField; - private String variantOptionsTableName; + private Boolean usesVariants = false; + private BackendVariantsConfig backendVariantsConfig; // todo - at some point, we may want to apply this to secret properties on subclasses? // @JsonFilter("secretsFilter") + @Deprecated(since = "Replaced by filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") + private String variantOptionsTableTypeField; // a field on which to filter the variant-options table, to limit which records in it are available as variants + + @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") + private String variantOptionsTableTypeValue; // value for the type-field, to limit which records in it are available as variants; but also, the key in the session.backendVariants map! + /******************************************************************************* @@ -394,22 +396,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface - /******************************************************************************* - ** Getter for variantOptionsTableIdField - *******************************************************************************/ - public String getVariantOptionsTableIdField() - { - return (this.variantOptionsTableIdField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableIdField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key") public void setVariantOptionsTableIdField(String variantOptionsTableIdField) { - this.variantOptionsTableIdField = variantOptionsTableIdField; + ///////////////////////////////////////////////// + // noop as we migrate to backendVariantsConfig // + ///////////////////////////////////////////////// } @@ -417,30 +412,24 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableIdField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key") public QBackendMetaData withVariantOptionsTableIdField(String variantOptionsTableIdField) { - this.variantOptionsTableIdField = variantOptionsTableIdField; + this.setVariantOptionsTableIdField(variantOptionsTableIdField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableNameField - *******************************************************************************/ - public String getVariantOptionsTableNameField() - { - return (this.variantOptionsTableNameField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableNameField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel") public void setVariantOptionsTableNameField(String variantOptionsTableNameField) { - this.variantOptionsTableNameField = variantOptionsTableNameField; + ///////////////////////////////////////////////// + // noop as we migrate to backendVariantsConfig // + ///////////////////////////////////////////////// } @@ -448,30 +437,26 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableNameField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel") public QBackendMetaData withVariantOptionsTableNameField(String variantOptionsTableNameField) { - this.variantOptionsTableNameField = variantOptionsTableNameField; + this.setVariantOptionsTableNameField(variantOptionsTableNameField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableTypeField - *******************************************************************************/ - public String getVariantOptionsTableTypeField() - { - return (this.variantOptionsTableTypeField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableTypeField *******************************************************************************/ + @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") public void setVariantOptionsTableTypeField(String variantOptionsTableTypeField) { this.variantOptionsTableTypeField = variantOptionsTableTypeField; + if(this.variantOptionsTableTypeValue != null) + { + this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue))); + } } @@ -479,30 +464,28 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableTypeField *******************************************************************************/ + @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") public QBackendMetaData withVariantOptionsTableTypeField(String variantOptionsTableTypeField) { - this.variantOptionsTableTypeField = variantOptionsTableTypeField; + this.setVariantOptionsTableTypeField(variantOptionsTableTypeField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableTypeValue - *******************************************************************************/ - public String getVariantOptionsTableTypeValue() - { - return (this.variantOptionsTableTypeValue); - } - - - /******************************************************************************* ** Setter for variantOptionsTableTypeValue *******************************************************************************/ + @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") public void setVariantOptionsTableTypeValue(String variantOptionsTableTypeValue) { + this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeValue); + this.variantOptionsTableTypeValue = variantOptionsTableTypeValue; + if(this.variantOptionsTableTypeField != null) + { + this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue))); + } } @@ -510,30 +493,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableTypeValue *******************************************************************************/ + @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") public QBackendMetaData withVariantOptionsTableTypeValue(String variantOptionsTableTypeValue) { - this.variantOptionsTableTypeValue = variantOptionsTableTypeValue; + this.setVariantOptionsTableTypeValue(variantOptionsTableTypeValue); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableUsernameField - *******************************************************************************/ - public String getVariantOptionsTableUsernameField() - { - return (this.variantOptionsTableUsernameField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableUsernameField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableUsernameField(String variantOptionsTableUsernameField) { - this.variantOptionsTableUsernameField = variantOptionsTableUsernameField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.USERNAME, variantOptionsTableUsernameField); } @@ -541,30 +516,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableUsernameField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableUsernameField(String variantOptionsTableUsernameField) { - this.variantOptionsTableUsernameField = variantOptionsTableUsernameField; + this.setVariantOptionsTableUsernameField(variantOptionsTableUsernameField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTablePasswordField - *******************************************************************************/ - public String getVariantOptionsTablePasswordField() - { - return (this.variantOptionsTablePasswordField); - } - - - /******************************************************************************* ** Setter for variantOptionsTablePasswordField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTablePasswordField(String variantOptionsTablePasswordField) { - this.variantOptionsTablePasswordField = variantOptionsTablePasswordField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.PASSWORD, variantOptionsTablePasswordField); } @@ -572,30 +539,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTablePasswordField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTablePasswordField(String variantOptionsTablePasswordField) { - this.variantOptionsTablePasswordField = variantOptionsTablePasswordField; + this.setVariantOptionsTablePasswordField(variantOptionsTablePasswordField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableApiKeyField - *******************************************************************************/ - public String getVariantOptionsTableApiKeyField() - { - return (this.variantOptionsTableApiKeyField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableApiKeyField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField) { - this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.API_KEY, variantOptionsTableApiKeyField); } @@ -603,30 +562,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableApiKeyField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField) { - this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField; + this.setVariantOptionsTableApiKeyField(variantOptionsTableApiKeyField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableName - *******************************************************************************/ - public String getVariantOptionsTableName() - { - return (this.variantOptionsTableName); - } - - - /******************************************************************************* ** Setter for variantOptionsTableName *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.tableName") public void setVariantOptionsTableName(String variantOptionsTableName) { - this.variantOptionsTableName = variantOptionsTableName; + this.getOrWithNewBackendVariantsConfig().withOptionsTableName(variantOptionsTableName); } @@ -634,9 +585,10 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableName *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.tableName") public QBackendMetaData withVariantOptionsTableName(String variantOptionsTableName) { - this.variantOptionsTableName = variantOptionsTableName; + this.setVariantOptionsTableName(variantOptionsTableName); return (this); } @@ -651,22 +603,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface qInstance.addBackend(this); } - /******************************************************************************* - ** Getter for variantOptionsTableClientIdField - *******************************************************************************/ - public String getVariantOptionsTableClientIdField() - { - return (this.variantOptionsTableClientIdField); - } - /******************************************************************************* ** Setter for variantOptionsTableClientIdField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField) { - this.variantOptionsTableClientIdField = variantOptionsTableClientIdField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_ID, variantOptionsTableClientIdField); } @@ -674,30 +619,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableClientIdField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField) { - this.variantOptionsTableClientIdField = variantOptionsTableClientIdField; + this.setVariantOptionsTableClientIdField(variantOptionsTableClientIdField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableClientSecretField - *******************************************************************************/ - public String getVariantOptionsTableClientSecretField() - { - return (this.variantOptionsTableClientSecretField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableClientSecretField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField) { - this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_SECRET, variantOptionsTableClientSecretField); } @@ -705,11 +642,55 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableClientSecretField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField) { - this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField; + this.setVariantOptionsTableClientSecretField(variantOptionsTableClientSecretField); return (this); } + + /******************************************************************************* + ** Getter for backendVariantsConfig + *******************************************************************************/ + public BackendVariantsConfig getBackendVariantsConfig() + { + return (this.backendVariantsConfig); + } + + + + /******************************************************************************* + ** Setter for backendVariantsConfig + *******************************************************************************/ + public void setBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig) + { + this.backendVariantsConfig = backendVariantsConfig; + } + + + + /******************************************************************************* + ** Fluent setter for backendVariantsConfig + *******************************************************************************/ + public QBackendMetaData withBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig) + { + this.backendVariantsConfig = backendVariantsConfig; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private BackendVariantsConfig getOrWithNewBackendVariantsConfig() + { + if(backendVariantsConfig == null) + { + setBackendVariantsConfig(new BackendVariantsConfig()); + } + return backendVariantsConfig; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java index e80e2d68..185b0035 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java @@ -30,4 +30,5 @@ public enum AuditLevel NONE, RECORD, FIELD + // idea: only audit changes to fields, e.g., on edit. though, is that a different dimension than this? } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java index 6e98fdec..01b7832e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -86,7 +86,6 @@ public class QFrontendTableMetaData ////////////////////////////////////////////////////////////////////////////////// - /******************************************************************************* ** *******************************************************************************/ @@ -170,7 +169,7 @@ public class QFrontendTableMetaData if(backend != null && backend.getUsesVariants()) { usesVariants = true; - variantTableLabel = QContext.getQInstance().getTable(backend.getVariantOptionsTableName()).getLabel(); + variantTableLabel = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName()).getLabel(); } this.helpContents = tableMetaData.getHelpContent(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 93ae4f96..1df589d1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -712,6 +713,25 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData + /******************************************************************************* + ** Getter for sections + ** + *******************************************************************************/ + public QFieldSection getSection(String name) + { + for(QFieldSection qFieldSection : CollectionUtils.nonNullList(sections)) + { + if(qFieldSection.getName().equals(name)) + { + return (qFieldSection); + } + } + + return (null); + } + + + /******************************************************************************* ** Setter for sections ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java new file mode 100644 index 00000000..52f84d78 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java @@ -0,0 +1,221 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; + + +/******************************************************************************* + ** Factory class for creating "standard" qfield sections. e.g., if you want + ** the same t1, t2, and t3 section on all your tables, use this class to + ** produce them. + ** + ** You can change the default name & iconNames for those sections, but note, + ** this is a static/utility style class, so those settings are static fields. + ** + ** The method customT2 is provided as not much of a shortcut over "doing it yourself", + ** but to allow all sections for a table to be produced through calls to this factory, + ** so they look more similar. + *******************************************************************************/ +public class SectionFactory +{ + private static String defaultT1name = "identity"; + private static String defaultT1iconName = "badge"; + private static String defaultT2name = "data"; + private static String defaultT2iconName = "text_snippet"; + private static String defaultT3name = "dates"; + private static String defaultT3iconName = "calendar_month"; + + + /******************************************************************************* + ** private constructor, to enforce static usage, e.g., to make clear the fields + ** are static fields. + ** + *******************************************************************************/ + private SectionFactory() + { + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection defaultT1(String... fieldNames) + { + return new QFieldSection(defaultT1name, new QIcon().withName(defaultT1iconName), Tier.T1, List.of(fieldNames)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection defaultT2(String... fieldNames) + { + return new QFieldSection(defaultT2name, new QIcon().withName(defaultT2iconName), Tier.T2, List.of(fieldNames)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection customT2(String name, QIcon icon, String... fieldNames) + { + return new QFieldSection(name, icon, Tier.T2, List.of(fieldNames)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection defaultT3(String... fieldNames) + { + return new QFieldSection(defaultT3name, new QIcon().withName(defaultT3iconName), Tier.T3, List.of(fieldNames)); + } + + + + /******************************************************************************* + ** Getter for defaultT1name + *******************************************************************************/ + public static String getDefaultT1name() + { + return (SectionFactory.defaultT1name); + } + + + + /******************************************************************************* + ** Setter for defaultT1name + *******************************************************************************/ + public static void setDefaultT1name(String defaultT1name) + { + SectionFactory.defaultT1name = defaultT1name; + } + + + + /******************************************************************************* + ** Getter for defaultT1iconName + *******************************************************************************/ + public static String getDefaultT1iconName() + { + return (SectionFactory.defaultT1iconName); + } + + + + /******************************************************************************* + ** Setter for defaultT1iconName + *******************************************************************************/ + public static void setDefaultT1iconName(String defaultT1iconName) + { + SectionFactory.defaultT1iconName = defaultT1iconName; + } + + + + /******************************************************************************* + ** Getter for defaultT2name + *******************************************************************************/ + public static String getDefaultT2name() + { + return (SectionFactory.defaultT2name); + } + + + + /******************************************************************************* + ** Setter for defaultT2name + *******************************************************************************/ + public static void setDefaultT2name(String defaultT2name) + { + SectionFactory.defaultT2name = defaultT2name; + } + + + + /******************************************************************************* + ** Getter for defaultT2iconName + *******************************************************************************/ + public static String getDefaultT2iconName() + { + return (SectionFactory.defaultT2iconName); + } + + + + /******************************************************************************* + ** Setter for defaultT2iconName + *******************************************************************************/ + public static void setDefaultT2iconName(String defaultT2iconName) + { + SectionFactory.defaultT2iconName = defaultT2iconName; + } + + + + /******************************************************************************* + ** Getter for defaultT3name + *******************************************************************************/ + public static String getDefaultT3name() + { + return (SectionFactory.defaultT3name); + } + + + + /******************************************************************************* + ** Setter for defaultT3name + *******************************************************************************/ + public static void setDefaultT3name(String defaultT3name) + { + SectionFactory.defaultT3name = defaultT3name; + } + + + + /******************************************************************************* + ** Getter for defaultT3iconName + *******************************************************************************/ + public static String getDefaultT3iconName() + { + return (SectionFactory.defaultT3iconName); + } + + + + /******************************************************************************* + ** Setter for defaultT3iconName + *******************************************************************************/ + public static void setDefaultT3iconName(String defaultT3iconName) + { + SectionFactory.defaultT3iconName = defaultT3iconName; + } + + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java new file mode 100644 index 00000000..ea57a3ed --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java @@ -0,0 +1,31 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +/******************************************************************************* + ** interface to be implemented by enums (presumably) that define the possible + ** settings a particular backend type can get from a variant record. + *******************************************************************************/ +public interface BackendVariantSetting +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java new file mode 100644 index 00000000..290c099f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java @@ -0,0 +1,225 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.util.HashMap; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** Configs for how a backend that uses variants works. Specifically: + ** + ** - the variant "type key" - e.g., key for variants map in session. + ** - what table supplies the variant options (optionsTableName + ** - an optional filter to apply to that options table + ** - a map of the settings that a backend gets from its variant table to the + ** field names in that table that they come from. e.g., a backend may have a + ** username attribute, whose value comes from a field named "theUser" in the + ** variant options table. + ** - an optional code reference to a variantRecordLookupFunction - to customize + ** how the variant record is looked up (such as, adding joined or other custom + ** fields). + *******************************************************************************/ +public class BackendVariantsConfig +{ + private String variantTypeKey; + + private String optionsTableName; + private QQueryFilter optionsFilter; + private QCodeReference variantRecordLookupFunction; + + private Map backendSettingSourceFieldNameMap; + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getOptionsTableName() + { + return (this.optionsTableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setOptionsTableName(String optionsTableName) + { + this.optionsTableName = optionsTableName; + } + + + + /******************************************************************************* + ** Getter for filter + *******************************************************************************/ + public QQueryFilter getOptionsFilter() + { + return (this.optionsFilter); + } + + + + /******************************************************************************* + ** Setter for filter + *******************************************************************************/ + public void setOptionsFilter(QQueryFilter optionsFilter) + { + this.optionsFilter = optionsFilter; + } + + + + /******************************************************************************* + ** Getter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public Map getBackendSettingSourceFieldNameMap() + { + return (this.backendSettingSourceFieldNameMap); + } + + + + /******************************************************************************* + ** Setter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public void setBackendSettingSourceFieldNameMap(Map backendSettingSourceFieldNameMap) + { + this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap; + } + + + + /******************************************************************************* + ** Fluent setter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public BackendVariantsConfig withBackendSettingSourceFieldName(BackendVariantSetting backendVariantSetting, String sourceFieldName) + { + if(this.backendSettingSourceFieldNameMap == null) + { + this.backendSettingSourceFieldNameMap = new HashMap<>(); + } + this.backendSettingSourceFieldNameMap.put(backendVariantSetting, sourceFieldName); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public BackendVariantsConfig withBackendSettingSourceFieldNameMap(Map backendSettingSourceFieldNameMap) + { + this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for variantTypeKey + *******************************************************************************/ + public String getVariantTypeKey() + { + return (this.variantTypeKey); + } + + + + /******************************************************************************* + ** Setter for variantTypeKey + *******************************************************************************/ + public void setVariantTypeKey(String variantTypeKey) + { + this.variantTypeKey = variantTypeKey; + } + + + + /******************************************************************************* + ** Fluent setter for variantTypeKey + *******************************************************************************/ + public BackendVariantsConfig withVariantTypeKey(String variantTypeKey) + { + this.variantTypeKey = variantTypeKey; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for optionsTableName + *******************************************************************************/ + public BackendVariantsConfig withOptionsTableName(String optionsTableName) + { + this.optionsTableName = optionsTableName; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for optionsFilter + *******************************************************************************/ + public BackendVariantsConfig withOptionsFilter(QQueryFilter optionsFilter) + { + this.optionsFilter = optionsFilter; + return (this); + } + + + /******************************************************************************* + ** Getter for variantRecordLookupFunction + *******************************************************************************/ + public QCodeReference getVariantRecordLookupFunction() + { + return (this.variantRecordLookupFunction); + } + + + + /******************************************************************************* + ** Setter for variantRecordLookupFunction + *******************************************************************************/ + public void setVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction) + { + this.variantRecordLookupFunction = variantRecordLookupFunction; + } + + + + /******************************************************************************* + ** Fluent setter for variantRecordLookupFunction + *******************************************************************************/ + public BackendVariantsConfig withVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction) + { + this.variantRecordLookupFunction = variantRecordLookupFunction; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java new file mode 100644 index 00000000..a94fbc4e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.io.Serializable; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; + + +/******************************************************************************* + ** Utility methods for backends working with Variants. + *******************************************************************************/ +public class BackendVariantsUtil +{ + + /******************************************************************************* + ** Get the variant id from the session for the backend. + *******************************************************************************/ + public static Serializable getVariantId(QBackendMetaData backendMetaData) throws QException + { + QSession session = QContext.getQSession(); + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) + { + throw (new QException("Could not find Backend Variant information in session under key '" + variantTypeKey + "' for Backend '" + backendMetaData.getName() + "'")); + } + Serializable variantId = session.getBackendVariants().get(variantTypeKey); + return variantId; + } + + + + /******************************************************************************* + ** For backends that use variants, look up the variant record (in theory, based + ** on an id in the session's backend variants map, then fetched from the backend's + ** variant options table. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException + { + Serializable variantId = getVariantId(backendMetaData); + + QRecord record; + if(backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction() != null) + { + Object o = QCodeLoader.getAdHoc(Object.class, backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction()); + if(o instanceof UnsafeFunction unsafeFunction) + { + record = ((UnsafeFunction) unsafeFunction).apply(variantId); + } + else if(o instanceof Function function) + { + record = ((Function) function).apply(variantId); + } + else + { + throw (new QException("Backend Variant's recordLookupFunction is not of any expected type (should have been caught by instance validation??)")); + } + } + else + { + GetInput getInput = new GetInput(); + getInput.setShouldMaskPasswords(false); + getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + getInput.setPrimaryKey(variantId); + GetOutput getOutput = new GetAction().execute(getInput); + + record = getOutput.getRecord(); + } + + if(record == null) + { + throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); + } + return record; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java new file mode 100644 index 00000000..59ccc38c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +/******************************************************************************* + ** temporary class, while we migrate from original way that variants were set up + ** e.g., by calling 'variantOptionsTableUsernameField', to the new way, using + ** the BackendVariantsConfig which uses a map of enum constants. + ** + ** so when those deprecated setters are removed, this enum can be too. + *****************************************************************************/ +public enum LegacyBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + API_KEY, + CLIENT_ID, + CLIENT_SECRET +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index 694ae332..dabfb3f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -55,6 +55,8 @@ public class BulkInsertExtractStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); + int rowsAdded = 0; int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java index c8b73ec2..b35bb24b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java @@ -22,16 +22,38 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* ** *******************************************************************************/ -public class BulkInsertLoadStep extends LoadViaInsertStep +public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSummaryProviderInterface { + private static final QLogger LOG = QLogger.getLogger(BulkInsertLoadStep.class); + + private Serializable firstInsertedPrimaryKey = null; + private Serializable lastInsertedPrimaryKey = null; + + /******************************************************************************* ** @@ -42,4 +64,62 @@ public class BulkInsertLoadStep extends LoadViaInsertStep return (QInputSource.USER); } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + super.runOnePage(runBackendStepInput, runBackendStepOutput); + + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName")); + + List insertedRecords = runBackendStepOutput.getRecords(); + for(QRecord insertedRecord : insertedRecords) + { + if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors())) + { + if(firstInsertedPrimaryKey == null) + { + firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + } + + lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + ArrayList processSummary = getTransformStep().getProcessSummary(runBackendStepOutput, isForResultScreen); + + try + { + if(firstInsertedPrimaryKey != null) + { + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepOutput.getValueString("tableName")); + QFieldMetaData field = table.getField(table.getPrimaryKeyField()); + if(field.getType().isNumeric()) + { + ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey); + idsLine.setCount(null); + processSummary.add(idsLine); + } + } + } + catch(Exception e) + { + LOG.warn("Error adding inserted-keys process summary line", e); + } + + return (processSummary); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index afe3c9f1..94032fe5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -32,16 +32,20 @@ import java.util.Set; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.BooleanUtils; @@ -63,12 +67,37 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep { try { - BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + QRecord savedBulkLoadProfileRecord = BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); - /////////////////////////////////////////////////////////////////// - // read process values - construct a bulkLoadProfile out of them // - /////////////////////////////////////////////////////////////////// - BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + BulkLoadProfile bulkLoadProfile; + if(BulkInsertStepUtils.isHeadless(runBackendStepInput)) + { + ////////////////////////////////////////////////////////////////////////////// + // if running headless, build bulkLoadProfile from the saved profile record // + ////////////////////////////////////////////////////////////////////////////// + if(savedBulkLoadProfileRecord == null) + { + throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load")); + } + + SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord); + + try + { + bulkLoadProfile = JsonUtils.toObject(savedBulkLoadProfile.getMappingJson(), BulkLoadProfile.class); + } + catch(Exception e) + { + throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e)); + } + } + else + { + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + } ///////////////////////////////////////////////////////////////////////// // put the list of bulk load profile into the process state - it's the // @@ -183,21 +212,32 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep ///////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); - if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + if(BulkInsertStepUtils.isHeadless(runBackendStepInput)) { - ////////////////////////////////////////////////////////////////////////////////// - // just go to the prepareValueMapping backend step - it'll figure out the rest. // - // it's also where the value-mapping loop of steps points. // - // and, this will actually be the default (e.g., the step after this one). // - ////////////////////////////////////////////////////////////////////////////////// - runBackendStepInput.addValue("valueMappingFieldIndex", -1); + //////////////////////////////////////////////////////////////////////// + // if running headless, always go straight to the preview screen next // + // todo actually, we could make this execute, right? // + //////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); } else { - ////////////////////////////////////////////////////////////////////////////////// - // else - if no values to map - continue with the standard streamed-ETL preview // - ////////////////////////////////////////////////////////////////////////////////// - BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + { + ////////////////////////////////////////////////////////////////////////////////// + // just go to the prepareValueMapping backend step - it'll figure out the rest. // + // it's also where the value-mapping loop of steps points. // + // and, this will actually be the default (e.g., the step after this one). // + ////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.addValue("valueMappingFieldIndex", -1); + } + else + { + ////////////////////////////////////////////////////////////////////////////////// + // else - if no values to map - continue with the standard streamed-ETL preview // + ////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + } } } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index b0db3ad0..e9c09cea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -29,11 +29,13 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerKeyRecordMessage; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.BooleanUtils; import org.json.JSONArray; @@ -69,6 +71,18 @@ public class BulkInsertStepUtils + /*************************************************************************** + ** + ***************************************************************************/ + public static void setStorageInputForTheFile(RunProcessInput runProcessInput, StorageInput storageInput) + { + ArrayList storageInputs = new ArrayList<>(); + storageInputs.add(storageInput); + runProcessInput.addValue("theFile", storageInputs); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -144,13 +158,62 @@ public class BulkInsertStepUtils /*************************************************************************** ** ***************************************************************************/ - public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public static QRecord handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId"); if(savedBulkLoadProfileId != null) { QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId); runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord); + return (savedBulkLoadProfileRecord); } + + return (null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static boolean isHeadless(RunBackendStepInput runBackendStepInput) + { + return (runBackendStepInput.getValuePrimitiveBoolean("isHeadless")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setHeadless(RunProcessInput runProcessInput) + { + runProcessInput.addValue("isHeadless", true); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setProcessTracerKeyRecordMessage(RunProcessInput runProcessInput, ProcessTracerKeyRecordMessage processTracerKeyRecordMessage) + { + runProcessInput.addValue("processTracerKeyRecordMessage", processTracerKeyRecordMessage); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProcessTracerKeyRecordMessage getProcessTracerKeyRecordMessage(RunBackendStepInput runBackendStepInput) + { + Serializable value = runBackendStepInput.getValue("processTracerKeyRecordMessage"); + if(value instanceof ProcessTracerKeyRecordMessage processTracerKeyRecordMessage) + { + return (processTracerKeyRecordMessage); + } + + return (null); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java index 626b8756..a0f90110 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -258,6 +258,11 @@ public class BulkLoadValueMapper valuesNotFound.add(value); } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - we should probably be doing a lot of what QJavalinImplementation.finishPossibleValuesRequest does here // + // to apply possible-value filters. difficult to pass values in, but needed... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + searchPossibleValueSourceInput.setIdList(idList); searchPossibleValueSourceInput.setLimit(values.size()); LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0))); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index cbe1e1a9..47c8c036 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -239,6 +239,10 @@ public class ColumnStatsStep implements BackendStep QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(); qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, queryJoin == null ? null : List.of(queryJoin), null); + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // todo - be aware of possible name collisions here!! (e.g., a table w/ a field named `count`) // + ///////////////////////////////////////////////////////////////////////////////////////////////// QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "count", countField), valueCounts); runBackendStepOutput.addValue("valueCounts", valueCounts); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java new file mode 100644 index 00000000..53938092 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java @@ -0,0 +1,66 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +/******************************************************************************* + ** Specialization of process tracer message, to indicate a 'key record' that was + ** used as an input or trigger to a process. + *******************************************************************************/ +public class ProcessTracerKeyRecordMessage extends ProcessTracerMessage +{ + private final String tableName; + private final Integer recordId; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public ProcessTracerKeyRecordMessage(String tableName, Integer recordId) + { + super("Process Key Record is " + tableName + " " + recordId); + this.tableName = tableName; + this.recordId = recordId; + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Getter for recordId + *******************************************************************************/ + public Integer getRecordId() + { + return (this.recordId); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java index df2dd558..bb27a0e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java @@ -22,13 +22,16 @@ package com.kingsrook.qqq.backend.core.processes.tracing; +import java.io.Serializable; + + /******************************************************************************* ** Basic class that can be passed in to ProcessTracerInterface.handleMessage. ** This class just provides for a string message. We anticipate subclasses ** that may have more specific data, that specific tracer implementations may ** be aware of. *******************************************************************************/ -public class ProcessTracerMessage +public class ProcessTracerMessage implements Serializable { private String message; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index 3dfec3ae..88246600 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -441,11 +441,16 @@ public class QScheduleManager try { HashMap parameters = new HashMap<>(paramMap); - HashMap variantMap = new HashMap<>(Map.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + String variantOptionsTableName = backendMetaData.getBackendVariantsConfig().getOptionsTableName(); + String variantOptionsTableIdFieldName = QContext.getQInstance().getTable(variantOptionsTableName).getPrimaryKeyField(); + + HashMap variantMap = new HashMap<>(Map.of(variantTypeKey, qRecord.getValue(variantOptionsTableIdFieldName))); parameters.put("backendVariantData", variantMap); - String identity = schedulableIdentity.getIdentity() + ";" + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); - String description = schedulableIdentity.getDescription() + " for variant: " + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); + String identity = schedulableIdentity.getIdentity() + ";" + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName); + String description = schedulableIdentity.getDescription() + " for variant: " + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName); BasicSchedulableIdentity variantIdentity = new BasicSchedulableIdentity(identity, description); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java index c7cc6a4c..e7b46ea3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java @@ -34,9 +34,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -102,7 +99,8 @@ public class SchedulerUtils try { QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend()); - Map thisVariantData = MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())); + QTableMetaData variantTable = QContext.getQInstance().getTable(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + Map thisVariantData = MapBuilder.of(backendMetaData.getBackendVariantsConfig().getVariantTypeKey(), qRecord.getValue(variantTable.getPrimaryKeyField())); executeSingleProcess(process, thisVariantData, processInputValues); } catch(Exception e) @@ -181,8 +179,8 @@ public class SchedulerUtils QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(processMetaData.getVariantBackend()); QueryInput queryInput = new QueryInput(); - queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue()))); + queryInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + queryInput.setFilter(backendMetaData.getBackendVariantsConfig().getOptionsFilter()); QueryOutput queryOutput = new QueryAction().execute(queryInput); records = queryOutput.getRecords(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index c09d1ac8..8419b05a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -55,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; @@ -93,12 +95,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -182,6 +187,143 @@ public class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBackendVariants() + { + BackendVariantSetting setting = new BackendVariantSetting() {}; + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true)), + "Missing backendVariantsConfig in backend [variant] which is marked as usesVariants"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(false) + .withBackendVariantsConfig(new BackendVariantsConfig())), + "Should not have a backendVariantsConfig"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(null) + .withBackendVariantsConfig(new BackendVariantsConfig())), + "Should not have a backendVariantsConfig"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig())), + "Missing variantTypeKey in backendVariantsConfig", + "Missing optionsTableName in backendVariantsConfig", + "Missing or empty backendSettingSourceFieldNameMap"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName("notATable") + .withBackendSettingSourceFieldNameMap(Map.of(setting, "field")))), + "Unrecognized optionsTableName [notATable] in backendVariantsConfig"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withOptionsFilter(new QQueryFilter(new QFilterCriteria("notAField", QCriteriaOperator.EQUALS, 1))) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")))), + "optionsFilter in backendVariantsConfig in backend [variant]: Criteria fieldName notAField is not a field"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "noSuchField")))), + "Unrecognized fieldName [noSuchField] in backendSettingSourceFieldNameMap"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(CustomizerThatIsNotOfTheRightBaseClass.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + )), + "VariantRecordSupplier in backendVariantsConfig in backend [variant]: CodeReference is not any of the expected types: com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction, java.util.function.Function"); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")) + ))); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(VariantRecordFunction.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + ))); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(VariantRecordUnsafeFunction.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + ))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class VariantRecordFunction implements Function + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QRecord apply(Serializable serializable) + { + return null; + } + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class VariantRecordUnsafeFunction implements UnsafeFunction + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QRecord apply(Serializable serializable) throws QException + { + return null; + } + } + + + /******************************************************************************* ** Test an instance with null tables - should throw. ** @@ -2369,7 +2511,7 @@ public class QInstanceValidatorTest extends BaseTest { int noOfReasons = actualReasons == null ? 0 : actualReasons.size(); assertEquals(expectedReasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", expectedReasons) - + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--")); + + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--")); } for(String reason : expectedReasons) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java new file mode 100644 index 00000000..118c27d3 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for EmptyMetaDataProducerOutput + *******************************************************************************/ +class EmptyMetaDataProducerOutputTest +{ + + /******************************************************************************* + ** sorry, just here to avoid a dip in coverage. + *******************************************************************************/ + @Test + void test() + { + QInstance qInstance = new QInstance(); + new EmptyMetaDataProducerOutput().addSelfToInstance(qInstance); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java new file mode 100644 index 00000000..5f406acb --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for SectionFactory + *******************************************************************************/ +class SectionFactoryTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + QFieldSection t1section = SectionFactory.defaultT1("id", "name"); + assertEquals(SectionFactory.getDefaultT1name(), t1section.getName()); + assertEquals(SectionFactory.getDefaultT1iconName(), t1section.getIcon().getName()); + assertEquals(Tier.T1, t1section.getTier()); + assertEquals(List.of("id", "name"), t1section.getFieldNames()); + + QFieldSection t2section = SectionFactory.defaultT2("size", "age"); + assertEquals(SectionFactory.getDefaultT2name(), t2section.getName()); + assertEquals(SectionFactory.getDefaultT2iconName(), t2section.getIcon().getName()); + assertEquals(Tier.T2, t2section.getTier()); + assertEquals(List.of("size", "age"), t2section.getFieldNames()); + + QFieldSection t3section = SectionFactory.defaultT3("createDate", "modifyDate"); + assertEquals(SectionFactory.getDefaultT3name(), t3section.getName()); + assertEquals(SectionFactory.getDefaultT3iconName(), t3section.getIcon().getName()); + assertEquals(Tier.T3, t3section.getTier()); + assertEquals(List.of("createDate", "modifyDate"), t3section.getFieldNames()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java new file mode 100644 index 00000000..5957ee61 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java @@ -0,0 +1,100 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for BackendVariantsUtil + *******************************************************************************/ +class BackendVariantsUtilTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetVariantId() throws QException + { + QBackendMetaData myBackend = getBackendMetaData(); + + assertThatThrownBy(() -> BackendVariantsUtil.getVariantId(myBackend)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701)); + assertEquals(1701, BackendVariantsUtil.getVariantId(myBackend)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData getBackendMetaData() + { + QBackendMetaData myBackend = new QBackendMetaData() + .withName("TestBackend") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withOptionsTableName(TestUtils.TABLE_NAME_SHAPE) + .withVariantTypeKey("yourSelectedShape")); + return myBackend; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetVariantRecord() throws QException + { + QBackendMetaData myBackend = getBackendMetaData(); + + TestUtils.insertDefaultShapes(QContext.getQInstance()); + + assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701)); + assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend)) + .hasMessageContaining("Could not find Backend Variant in table shape with id '1701'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1)); + QRecord variantRecord = BackendVariantsUtil.getVariantRecord(myBackend); + assertEquals(1, variantRecord.getValueInteger("id")); + assertNotNull(variantRecord.getValue("name")); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java index 8425fbfb..6a32ae17 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java @@ -61,7 +61,6 @@ public abstract class AbstractAPIAction apiActionUtil.setBackendMetaData(this.backendMetaData); apiActionUtil.setActionInput(actionInput); - apiActionUtil.setSession(session); } } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 209285e0..2d34e264 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -36,7 +36,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -61,6 +60,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -77,6 +79,7 @@ import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorExcep import com.kingsrook.qqq.backend.module.api.model.AuthorizationType; import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog; import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData; +import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendVariantSetting; import com.kingsrook.qqq.backend.module.api.model.metadata.APITableBackendDetails; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.BooleanUtils; @@ -114,7 +117,6 @@ public class BaseAPIActionUtil { private final QLogger LOG = QLogger.getLogger(BaseAPIActionUtil.class); - protected QSession session; // todo not commit - delete!! protected APIBackendMetaData backendMetaData; protected AbstractTableActionInput actionInput; @@ -777,8 +779,8 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); - return (record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField())); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); + return (record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.API_KEY, APIBackendVariantSetting.API_KEY))); } return (backendMetaData.getApiKey()); @@ -786,6 +788,18 @@ public class BaseAPIActionUtil + /*************************************************************************** + ** todo - once deprecated variant methods are removed from QBackendMetaData, + ** then we can remove the LegacyBackendVariantSetting enum, and this param. + ***************************************************************************/ + private String getVariantSettingSourceFieldName(APIBackendMetaData backendMetaData, LegacyBackendVariantSetting legacyBackendVariantSetting, APIBackendVariantSetting apiBackendVariantSetting) + { + Map map = CollectionUtils.nonNullMap(backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap()); + return map.getOrDefault(legacyBackendVariantSetting, map.get(apiBackendVariantSetting)); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -793,8 +807,11 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); - return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField()))); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); + return (Pair.of( + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.USERNAME, APIBackendVariantSetting.USERNAME)), + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.PASSWORD, APIBackendVariantSetting.PASSWORD)) + )); } return (Pair.of(backendMetaData.getUsername(), backendMetaData.getPassword())); @@ -802,46 +819,6 @@ public class BaseAPIActionUtil - /******************************************************************************* - ** For backends that use variants, look up the variant record (in theory, based - ** on an id in the session's backend variants map, then fetched from the backend's - ** variant options table. - *******************************************************************************/ - protected QRecord getVariantRecord() throws QException - { - Serializable variantId = getVariantId(); - GetInput getInput = new GetInput(); - getInput.setShouldMaskPasswords(false); - getInput.setTableName(backendMetaData.getVariantOptionsTableName()); - getInput.setPrimaryKey(variantId); - GetOutput getOutput = new GetAction().execute(getInput); - - QRecord record = getOutput.getRecord(); - if(record == null) - { - throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'")); - } - return record; - } - - - - /******************************************************************************* - ** Get the variant id from the session for the backend. - *******************************************************************************/ - protected Serializable getVariantId() throws QException - { - QSession session = QContext.getQSession(); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) - { - throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); - } - Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue()); - return variantId; - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -854,7 +831,7 @@ public class BaseAPIActionUtil String accessTokenKey = "accessToken"; if(backendMetaData.getUsesVariants()) { - Serializable variantId = getVariantId(); + Serializable variantId = BackendVariantsUtil.getVariantId(backendMetaData); accessTokenKey = accessTokenKey + ":" + variantId; } @@ -944,8 +921,11 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); - return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableClientIdField()), record.getValueString(backendMetaData.getVariantOptionsTableClientSecretField()))); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); + return (Pair.of( + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_ID, APIBackendVariantSetting.CLIENT_ID)), + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_SECRET, APIBackendVariantSetting.CLIENT_SECRET)) + )); } return (Pair.of(backendMetaData.getClientId(), backendMetaData.getClientSecret())); @@ -1480,9 +1460,9 @@ public class BaseAPIActionUtil ** Setter for session ** *******************************************************************************/ + @Deprecated(since = "wasn't used.") public void setSession(QSession session) { - this.session = session; } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java new file mode 100644 index 00000000..681977d3 --- /dev/null +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.api.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; + + +/******************************************************************************* + ** settings that the API backend module can get from a backend variant. + *******************************************************************************/ +public enum APIBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + API_KEY, + CLIENT_ID, + CLIENT_SECRET +} diff --git a/qqq-backend-module-filesystem/pom.xml b/qqq-backend-module-filesystem/pom.xml index 32cc274f..9344db4a 100644 --- a/qqq-backend-module-filesystem/pom.xml +++ b/qqq-backend-module-filesystem/pom.xml @@ -50,6 +50,17 @@ aws-java-sdk-s3 1.12.261 + + org.apache.sshd + sshd-sftp + 2.14.0 + + + org.apache.sshd + sshd-sftp + 2.14.0 + + cloud.localstack localstack-utils @@ -57,6 +68,19 @@ test + + org.testcontainers + testcontainers + 1.15.3 + test + + + net.java.dev.jna + jna + 5.7.0 + test + + org.apache.maven.plugins diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index fdf343c6..0557e68a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -25,9 +25,13 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; @@ -38,6 +42,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -47,12 +53,19 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -68,6 +81,8 @@ public abstract class AbstractBaseFilesystemAction { private static final QLogger LOG = QLogger.getLogger(AbstractBaseFilesystemAction.class); + protected QRecord backendVariantRecord = null; + /******************************************************************************* @@ -80,6 +95,21 @@ public abstract class AbstractBaseFilesystemAction + /*************************************************************************** + ** get the size of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Long getFileSize(FILE file); + + /*************************************************************************** + ** get the createDate of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Instant getFileCreateDate(FILE file); + + /*************************************************************************** + ** get the createDate of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Instant getFileModifyDate(FILE file); + /******************************************************************************* ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses. *******************************************************************************/ @@ -116,13 +146,21 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; + + /******************************************************************************* ** e.g., with a base path of /foo/ ** and a table path of /bar/ ** and a file at /foo/bar/baz.txt ** give us just the baz.txt part. *******************************************************************************/ - public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable); + public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); + String withoutLeadingSlash = stripLeadingSlash(strippedPath); // todo - dangerous, do all backends really want this?? + return (withoutLeadingSlash); + } @@ -133,7 +171,17 @@ public abstract class AbstractBaseFilesystemAction public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase) { AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase); - String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : ""; + + String basePath = metaData.getBasePath(); + if(backendBase.getUsesVariants()) + { + Map fieldNameMap = backendBase.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap(); + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.BASE_PATH)) + { + basePath = backendVariantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.BASE_PATH)); + } + } + String fullPath = StringUtils.hasContent(basePath) ? basePath : ""; AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); if(StringUtils.hasContent(tableDetails.getBasePath())) @@ -164,6 +212,34 @@ public abstract class AbstractBaseFilesystemAction + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripLeadingSlash(String path) + { + if(path == null) + { + return (null); + } + return (path.replaceFirst("^/+", "")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripTrailingSlash(String path) + { + if(path == null) + { + return (null); + } + return (path.replaceFirst("/+$", "")); + } + + + /******************************************************************************* ** Get the backend metaData, type-checked as the requested type. *******************************************************************************/ @@ -208,106 +284,219 @@ public abstract class AbstractBaseFilesystemAction AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); List files = listFiles(table, queryInput.getBackend(), queryInput.getFilter()); - int recordCount = 0; - - FILE_LOOP: - for(FILE file : files) + switch(tableDetails.getCardinality()) { - InputStream inputStream = readFile(file); - switch(tableDetails.getCardinality()) - { - case MANY: - { - LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file))); - switch(tableDetails.getRecordFormat()) - { - case CSV: - { - String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - fileContents = customizeFileContentsAfterReading(table, fileContents); - - if(queryInput.getRecordPipe() != null) - { - new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> - { - //////////////////////////////////////////////////////////////////////////////////////////// - // Before the records go into the pipe, make sure their backend details are added to them // - //////////////////////////////////////////////////////////////////////////////////////////// - addBackendDetailsToRecord(record, file); - })); - } - else - { - List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - queryOutput.addRecords(recordsInFile); - } - break; - } - case JSON: - { - String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - fileContents = customizeFileContentsAfterReading(table, fileContents); - - // todo - pipe support!! - List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - - queryOutput.addRecords(recordsInFile); - break; - } - default: - { - throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); - } - } - break; - } - case ONE: - { - //////////////////////////////////////////////////////////////////////////////// - // for one-record tables, put the entire file's contents into a single record // - //////////////////////////////////////////////////////////////////////////////// - String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); - byte[] bytes = inputStream.readAllBytes(); - - QRecord record = new QRecord() - .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase) - .withValue(tableDetails.getContentsFieldName(), bytes); - queryOutput.addRecord(record); - - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - // keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) // - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - recordCount++; - - //////////////////////////////////////////////////////////////////////////// - // break out of the file loop if we have hit the limit (if one was given) // - //////////////////////////////////////////////////////////////////////////// - if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null) - { - if(recordCount >= queryInput.getFilter().getLimit()) - { - break FILE_LOOP; - } - } - - break; - } - default: - { - throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); - } - } + case MANY -> completeExecuteQueryForManyTable(queryInput, queryOutput, files, table, tableDetails); + case ONE -> completeExecuteQueryForOneTable(queryInput, queryOutput, files, table, tableDetails); + default -> throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); } - return queryOutput; + return (queryOutput); } catch(Exception e) { LOG.warn("Error executing query", e); throw new QException("Error executing query", e); } + finally + { + postAction(); + } + } + + + /*************************************************************************** + ** + ***************************************************************************/ + private void setRecordValueIfFieldNameHasContent(QRecord record, String fieldName, UnsafeSupplier valueSupplier) + { + if(StringUtils.hasContent(fieldName)) + { + try + { + record.setValue(fieldName, valueSupplier.get()); + } + catch(Exception e) + { + LOG.warn("Error setting record value for field", e, logPair("fieldName", fieldName)); + } + } + } + + + /*************************************************************************** + ** + ***************************************************************************/ + private void completeExecuteQueryForOneTable(QueryInput queryInput, QueryOutput queryOutput, List files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException + { + int recordCount = 0; + List records = new ArrayList<>(); + + for(FILE file : files) + { + //////////////////////////////////////////////////////////////////////////////// + // for one-record tables, put the entire file's contents into a single record // + //////////////////////////////////////////////////////////////////////////////// + String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); + QRecord record = new QRecord(); + + setRecordValueIfFieldNameHasContent(record, tableDetails.getFileNameFieldName(), () -> filePathWithoutBase); + setRecordValueIfFieldNameHasContent(record, tableDetails.getBaseNameFieldName(), () -> stripAllPaths(filePathWithoutBase)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getSizeFieldName(), () -> getFileSize(file)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getCreateDateFieldName(), () -> getFileCreateDate(file)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getModifyDateFieldName(), () -> getFileModifyDate(file)); + + if(shouldHeavyFileContentsBeRead(queryInput, table, tableDetails)) + { + try(InputStream inputStream = readFile(file)) + { + byte[] bytes = inputStream.readAllBytes(); + record.withValue(tableDetails.getContentsFieldName(), bytes); + } + catch(Exception e) + { + record.addError(new SystemErrorStatusMessage("Error reading file contents: " + e.getMessage())); + } + } + else + { + Long size = record.getValueLong(tableDetails.getSizeFieldName()); + if(size != null) + { + if(record.getBackendDetails() == null) + { + record.setBackendDetails(new HashMap<>()); + } + + if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null) + { + record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>()); + } + + ((Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size); + } + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // the listFiles method may have used a "path" criteria. // + // if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + QQueryFilter filterForRecords = queryInput.getFilter(); + if(filterForRecords != null) + { + filterForRecords = filterForRecords.clone(); + + CollectionUtils.nonNullList(filterForRecords.getCriteria()) + .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria); + } + + if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record)) + { + records.add(record); + } + } + + BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), records); + records = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), records); + queryOutput.addRecords(records); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private Serializable stripAllPaths(String filePath) + { + if(filePath == null) + { + return null; + } + + return (filePath.replaceFirst(".*/", "")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static boolean isPathEqualsCriteria(QFilterCriteria criteria) + { + return "path".equals(criteria.getFieldName()) && QCriteriaOperator.EQUALS.equals(criteria.getOperator()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void completeExecuteQueryForManyTable(QueryInput queryInput, QueryOutput queryOutput, List files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException, IOException + { + int recordCount = 0; + + for(FILE file : files) + { + try(InputStream inputStream = readFile(file)) + { + LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file))); + switch(tableDetails.getRecordFormat()) + { + case CSV -> + { + String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + if(queryInput.getRecordPipe() != null) + { + new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> + { + //////////////////////////////////////////////////////////////////////////////////////////// + // Before the records go into the pipe, make sure their backend details are added to them // + //////////////////////////////////////////////////////////////////////////////////////////// + addBackendDetailsToRecord(record, file); + })); + } + else + { + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + queryOutput.addRecords(recordsInFile); + } + } + case JSON -> + { + String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + // todo - pipe support!! + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + + queryOutput.addRecords(recordsInFile); + } + default -> throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static boolean shouldHeavyFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) + { + boolean doReadContents = true; + if(table.getField(tableDetails.getContentsFieldName()).getIsHeavy()) + { + if(!queryInput.getShouldFetchHeavyFields()) + { + doReadContents = false; + } + } + return doReadContents; } @@ -319,7 +508,16 @@ public abstract class AbstractBaseFilesystemAction { QueryInput queryInput = new QueryInput(); queryInput.setTableName(countInput.getTableName()); - queryInput.setFilter(countInput.getFilter()); + + QQueryFilter filter = countInput.getFilter(); + if(filter != null) + { + filter = filter.clone(); + filter.setSkip(null); + filter.setLimit(null); + } + + queryInput.setFilter(filter); QueryOutput queryOutput = executeQuery(queryInput); CountOutput countOutput = new CountOutput(); @@ -353,14 +551,25 @@ public abstract class AbstractBaseFilesystemAction ** Method that subclasses can override to add pre-action things (e.g., setting up ** s3 client). *******************************************************************************/ - public void preAction(QBackendMetaData backendMetaData) + public void preAction(QBackendMetaData backendMetaData) throws QException { - ///////////////////////////////////////////////////////////////////// - // noop in base class - subclasses can add functionality if needed // - ///////////////////////////////////////////////////////////////////// + if(backendMetaData.getUsesVariants()) + { + this.backendVariantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData); + } } + /*************************************************************************** + ** Method that subclasses can override to add post-action things (e.g., closing resources) + ***************************************************************************/ + public void postAction() + { + ////////////////// + // noop in base // + ////////////////// + } + /******************************************************************************* ** @@ -411,10 +620,18 @@ public abstract class AbstractBaseFilesystemAction { for(QRecord record : insertInput.getRecords()) { - String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); - writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); - record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); - output.addRecord(record); + try + { + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); + writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); + record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); + output.addRecord(record); + } + catch(Exception e) + { + record.addError(new SystemErrorStatusMessage("Error writing file: " + e.getMessage())); + output.addRecord(record); + } } } else @@ -428,5 +645,10 @@ public abstract class AbstractBaseFilesystemAction { throw new QException("Error executing insert: " + e.getMessage(), e); } + finally + { + postAction(); + } } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index f50f1b4c..307849b6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -41,6 +41,10 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private String contentsFieldName; private String fileNameFieldName; + private String baseNameFieldName; + private String sizeFieldName; + private String createDateFieldName; + private String modifyDateFieldName; @@ -281,4 +285,128 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails } } + + /******************************************************************************* + ** Getter for sizeFieldName + *******************************************************************************/ + public String getSizeFieldName() + { + return (this.sizeFieldName); + } + + + + /******************************************************************************* + ** Setter for sizeFieldName + *******************************************************************************/ + public void setSizeFieldName(String sizeFieldName) + { + this.sizeFieldName = sizeFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for sizeFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withSizeFieldName(String sizeFieldName) + { + this.sizeFieldName = sizeFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDateFieldName + *******************************************************************************/ + public String getCreateDateFieldName() + { + return (this.createDateFieldName); + } + + + + /******************************************************************************* + ** Setter for createDateFieldName + *******************************************************************************/ + public void setCreateDateFieldName(String createDateFieldName) + { + this.createDateFieldName = createDateFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for createDateFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withCreateDateFieldName(String createDateFieldName) + { + this.createDateFieldName = createDateFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDateFieldName + *******************************************************************************/ + public String getModifyDateFieldName() + { + return (this.modifyDateFieldName); + } + + + + /******************************************************************************* + ** Setter for modifyDateFieldName + *******************************************************************************/ + public void setModifyDateFieldName(String modifyDateFieldName) + { + this.modifyDateFieldName = modifyDateFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDateFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withModifyDateFieldName(String modifyDateFieldName) + { + this.modifyDateFieldName = modifyDateFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for baseNameFieldName + *******************************************************************************/ + public String getBaseNameFieldName() + { + return (this.baseNameFieldName); + } + + + + /******************************************************************************* + ** Setter for baseNameFieldName + *******************************************************************************/ + public void setBaseNameFieldName(String baseNameFieldName) + { + this.baseNameFieldName = baseNameFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for baseNameFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withBaseNameFieldName(String baseNameFieldName) + { + this.baseNameFieldName = baseNameFieldName; + return (this); + } + + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java index 19c1601e..3c33eb39 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java @@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory; import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; /******************************************************************************* @@ -64,6 +70,7 @@ public class FilesystemTableMetaDataBuilder { case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails(); case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails(); + case SFTPBackendModule.BACKEND_TYPE -> new SFTPTableBackendDetails(); default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType()); }; @@ -72,12 +79,31 @@ public class FilesystemTableMetaDataBuilder .withIsHidden(true) .withBackendName(backend.getName()) .withPrimaryKeyField("fileName") - .withField(new QFieldMetaData("fileName", QFieldType.INTEGER)) - .withField(new QFieldMetaData("contents", QFieldType.STRING)) + + .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("baseName", QFieldType.STRING)) + .withField(new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB) + .withIsHeavy(true) + .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s") + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName") + )) + + .withSection(SectionFactory.defaultT1("fileName")) + .withSection(SectionFactory.defaultT2("baseName", "contents", "size")) + .withSection(SectionFactory.defaultT3("createDate", "modifyDate")) + .withBackendDetails(tableBackendDetails .withCardinality(Cardinality.ONE) .withFileNameFieldName("fileName") + .withBaseNameFieldName("baseName") .withContentsFieldName("contents") + .withSizeFieldName("size") + .withCreateDateFieldName("createDate") + .withModifyDateFieldName("modifyDate") .withBasePath(basePath) .withGlob(glob)); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index c9fb807f..5dde1ab6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -35,6 +35,8 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -61,6 +63,50 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Long getFileSize(File file) + { + return (file.length()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileCreateDate(File file) + { + try + { + Path path = file.toPath(); + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); + FileTime creationTime = attrs.creationTime(); + return creationTime.toInstant(); + } + catch(IOException e) + { + LOG.warn("Error getting file createDate", e, logPair("file", file)); + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileModifyDate(File file) + { + return Instant.ofEpochMilli(file.lastModified()); + } + + + /******************************************************************************* ** List the files for this table. *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java index 43867e99..f41ceb6b 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java @@ -63,7 +63,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -314,7 +313,7 @@ public class FilesystemImporterStep implements BackendStep /******************************************************************************* ** *******************************************************************************/ - private static void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException + private static void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws QException { if(removeFileAfterImport) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index 8a9a6272..05610fe6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; @@ -35,6 +36,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3CountAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; @@ -112,6 +114,17 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new S3CountAction(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 91383c4b..4c5e7b8c 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import java.io.IOException; import java.io.InputStream; +import java.time.Instant; import java.util.List; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; @@ -56,11 +57,44 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction. + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.AbstractSFTPAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPCountAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPDeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPInsertAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPStorageAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPUpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with SFTP filesystems (as a client) + *******************************************************************************/ +public class SFTPBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface +{ + public static final String BACKEND_TYPE = "sftp"; + + static + { + QBackendModuleDispatcher.registerBackendModule(new SFTPBackendModule()); + } + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractSFTPAction()); + } + + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return (BACKEND_TYPE); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (SFTPBackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return (SFTPTableBackendDetails.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new SFTPQueryAction(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new SFTPCountAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new SFTPInsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new SFTPUpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new SFTPDeleteAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return new SFTPStorageAction(); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java new file mode 100644 index 00000000..18cb6c70 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -0,0 +1,384 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import org.apache.sshd.client.SshClient; +import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.sftp.client.SftpClient; +import org.apache.sshd.sftp.client.SftpClientFactory; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Base class for all SFTP filesystem actions + *******************************************************************************/ +public class AbstractSFTPAction extends AbstractBaseFilesystemAction +{ + private static final QLogger LOG = QLogger.getLogger(AbstractSFTPAction.class); + + /*************************************************************************** + ** singleton implementing Initialization-on-Demand Holder idiom + ** to help ensure only a single SshClient object exists in a server. + ***************************************************************************/ + private static class SshClientManager + { + + /*************************************************************************** + ** + ***************************************************************************/ + private static class Holder + { + private static final SshClient INSTANCE = SshClient.setUpDefaultClient(); + + static + { + INSTANCE.start(); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static SshClient getInstance() + { + return Holder.INSTANCE; + } + } + + + + //////////////////////////////////////////////////////////////// + // open clientSessionFirst, then sftpClient // + // and close them in reverse (sftpClient, then clientSession) // + //////////////////////////////////////////////////////////////// + private ClientSession clientSession; + private SftpClient sftpClient; + + + + /******************************************************************************* + ** Set up the sftp utils object to be used for this action. + *******************************************************************************/ + @Override + public void preAction(QBackendMetaData backendMetaData) throws QException + { + super.preAction(backendMetaData); + + if(sftpClient != null) + { + LOG.debug("sftpClient object is already set - not re-setting it."); + return; + } + + try + { + SFTPBackendMetaData sftpBackendMetaData = getBackendMetaData(SFTPBackendMetaData.class, backendMetaData); + + String username = sftpBackendMetaData.getUsername(); + String password = sftpBackendMetaData.getPassword(); + String hostName = sftpBackendMetaData.getHostName(); + Integer port = sftpBackendMetaData.getPort(); + + if(backendMetaData.getUsesVariants()) + { + QRecord variantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData); + LOG.debug("Getting SFTP connection credentials from variant record", + logPair("tableName", backendMetaData.getBackendVariantsConfig().getOptionsTableName()), + logPair("id", variantRecord.getValue("id")), + logPair("name", variantRecord.getRecordLabel())); + Map fieldNameMap = backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap(); + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.USERNAME)) + { + username = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.USERNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PASSWORD)) + { + password = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.PASSWORD)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.HOSTNAME)) + { + hostName = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.HOSTNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PORT)) + { + port = variantRecord.getValueInteger(fieldNameMap.get(SFTPBackendVariantSetting.PORT)); + } + } + + makeConnection(username, hostName, port, password); + } + catch(IOException e) + { + throw (new QException("Error setting up SFTP connection", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void postAction() + { + Consumer closer = closable -> + { + if(closable != null) + { + try + { + closable.close(); + } + catch(Exception e) + { + LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName())); + } + } + }; + + closer.accept(sftpClient); + closer.accept(clientSession); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected SftpClient makeConnection(String username, String hostName, Integer port, String password) throws IOException + { + this.clientSession = SshClientManager.getInstance().connect(username, hostName, port).verify().getSession(); + clientSession.addPasswordIdentity(password); + clientSession.auth().verify(); + + this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); + return (this.sftpClient); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Long getFileSize(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getSize(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileCreateDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getCreateTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileModifyDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getModifyTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + { + try + { + String fullPath = getFullBasePath(table, backendBase); + + // todo - move somewhere shared + // todo - should all do this? + if(filter != null) + { + for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria())) + { + if(isPathEqualsCriteria(criteria)) + { + fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar); + } + } + } + + List rs = new ArrayList<>(); + + for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) + { + if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename())) + { + continue; + } + + if(dirEntry.getAttributes().isDirectory()) + { + // todo - recursive?? + continue; + } + + rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry)); + } + + return (rs); + } + catch(Exception e) + { + throw (new QException("Error listing files", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public InputStream readFile(SFTPDirEntryWithPath dirEntry) throws IOException + { + return (sftpClient.read(getFullPathForFile(dirEntry))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException + { + sftpClient.put(new ByteArrayInputStream(contents), path); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getFullPathForFile(SFTPDirEntryWithPath dirEntry) + { + return (dirEntry.path() + "/" + dirEntry.dirEntry().getFilename()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + throw (new QRuntimeException("Not yet implemented")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + throw (new QRuntimeException("Not yet implemented")); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected SftpClient getSftpClient(QBackendMetaData backend) throws QException + { + if(sftpClient == null) + { + preAction(backend); + } + + return (sftpClient); + } +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java new file mode 100644 index 00000000..f882662c --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountAction extends AbstractSFTPAction implements CountInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public CountOutput execute(CountInput countInput) throws QException + { + return (executeCount(countInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java new file mode 100644 index 00000000..365e3ec7 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteOutput execute(DeleteInput deleteInput) throws QException + { + throw new NotImplementedException("SFTP delete not implemented"); + /* + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java new file mode 100644 index 00000000..aa1c81dc --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertAction extends AbstractSFTPAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertOutput execute(InsertInput insertInput) throws QException + { + return (super.executeInsert(insertInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java new file mode 100644 index 00000000..5759c7ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPQueryAction extends AbstractSFTPAction implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryOutput execute(QueryInput queryInput) throws QException + { + return (super.executeQuery(queryInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java new file mode 100644 index 00000000..ddcb40f1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.utils.SFTPOutputStream; +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** (mass, streamed) storage action for sftp module + *******************************************************************************/ +public class SFTPStorageAction extends AbstractSFTPAction implements QStorageInterface +{ + + /******************************************************************************* + ** create an output stream in the storage backend - that can be written to, + ** for the purpose of inserting or writing a file into storage. + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + + SFTPOutputStream sftpOutputStream = new SFTPOutputStream(sftpClient, getFullPath(storageInput)); + return (sftpOutputStream); + } + catch(Exception e) + { + throw (new QException("Exception creating sftp output stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFullPath(StorageInput storageInput) throws QException + { + QTableMetaData table = storageInput.getTable(); + QBackendMetaData backend = storageInput.getBackend(); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference()); + + ///////////////////////////////////////////////////////////// + // s3 seems to do better w/o leading slashes, so, strip... // + ///////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = fullPath.substring(1); + } + + return fullPath; + } + + + + /******************************************************************************* + ** create an input stream in the storage backend - that can be read from, + ** for the purpose of getting or reading a file from storage. + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + InputStream inputStream = sftpClient.read(getFullPath(storageInput)); + + return (inputStream); + } + catch(Exception e) + { + throw (new QException("Exception getting sftp input stream for file.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDownloadURL(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + //S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + //preAction(backend); + // + //AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + //String fullPath = getFullPath(storageInput); + //return (amazonS3.getUrl(backend.getBucketName(), fullPath).toString()); + } + catch(Exception e) + { + throw (new QException("Exception getting the sftp download URL.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void makePublic(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + } + catch(Exception e) + { + throw (new QException("Exception making sftp file publicly available.", e)); + } + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java new file mode 100644 index 00000000..53b93ccf --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java @@ -0,0 +1,404 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** action for testing credentials for an SFTP backend connection + *******************************************************************************/ +public class SFTPTestConnectionAction extends AbstractSFTPAction +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public SFTPTestConnectionTestOutput testConnection(SFTPTestConnectionTestInput input) + { + try(SftpClient sftpClient = super.makeConnection(input.getUsername(), input.getHostName(), input.getPort(), input.getPassword())) + { + SFTPTestConnectionTestOutput output = new SFTPTestConnectionTestOutput().withIsConnectionSuccess(true); + + if(StringUtils.hasContent(input.basePath)) + { + try + { + Iterable dirEntries = sftpClient.readDir(input.basePath); + + ///////////////////////////////////////////////////////////////////////// + // it seems like only the .iterator call throws if bad directory here. // + ///////////////////////////////////////////////////////////////////////// + dirEntries.iterator(); + output.setIsListBasePathSuccess(true); + } + catch(Exception e) + { + output.setIsListBasePathSuccess(false); + output.setListBasePathErrorMessage(e.getMessage()); + } + } + + return output; + } + catch(Exception e) + { + return new SFTPTestConnectionTestOutput().withIsConnectionSuccess(false).withConnectionErrorMessage(e.getMessage()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class SFTPTestConnectionTestInput + { + private String username; + private String hostName; + private Integer port; + private String password; + private String basePath; + + + + /******************************************************************************* + ** Getter for username + ** + *******************************************************************************/ + public String getUsername() + { + return username; + } + + + + /******************************************************************************* + ** Setter for username + ** + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for hostName + ** + *******************************************************************************/ + public String getHostName() + { + return hostName; + } + + + + /******************************************************************************* + ** Setter for hostName + ** + *******************************************************************************/ + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + + + /******************************************************************************* + ** Fluent setter for hostName + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withHostName(String hostName) + { + this.hostName = hostName; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + ** + *******************************************************************************/ + public Integer getPort() + { + return port; + } + + + + /******************************************************************************* + ** Setter for port + ** + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent setter for port + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPort(Integer port) + { + this.port = port; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + ** + *******************************************************************************/ + public String getPassword() + { + return password; + } + + + + /******************************************************************************* + ** Setter for password + ** + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent setter for password + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Getter for basePath + ** + *******************************************************************************/ + public String getBasePath() + { + return basePath; + } + + + + /******************************************************************************* + ** Setter for basePath + ** + *******************************************************************************/ + public void setBasePath(String basePath) + { + this.basePath = basePath; + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withBasePath(String basePath) + { + this.basePath = basePath; + return (this); + } + + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class SFTPTestConnectionTestOutput + { + private Boolean isConnectionSuccess; + private String connectionErrorMessage; + + private Boolean isListBasePathSuccess; + private String listBasePathErrorMessage; + + + + /******************************************************************************* + ** Getter for isSuccess + ** + *******************************************************************************/ + public Boolean getIsConnectionSuccess() + { + return isConnectionSuccess; + } + + + + /******************************************************************************* + ** Setter for isSuccess + ** + *******************************************************************************/ + public void setIsConnectionSuccess(Boolean isSuccess) + { + this.isConnectionSuccess = isSuccess; + } + + + + /******************************************************************************* + ** Fluent setter for isSuccess + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withIsConnectionSuccess(Boolean isSuccess) + { + this.isConnectionSuccess = isSuccess; + return (this); + } + + + + /******************************************************************************* + ** Getter for connectionErrorMessage + ** + *******************************************************************************/ + public String getConnectionErrorMessage() + { + return connectionErrorMessage; + } + + + + /******************************************************************************* + ** Setter for connectionErrorMessage + ** + *******************************************************************************/ + public void setConnectionErrorMessage(String connectionErrorMessage) + { + this.connectionErrorMessage = connectionErrorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for connectionErrorMessage + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withConnectionErrorMessage(String connectionErrorMessage) + { + this.connectionErrorMessage = connectionErrorMessage; + return (this); + } + + + + /******************************************************************************* + ** Getter for listBasePathErrorMessage + *******************************************************************************/ + public String getListBasePathErrorMessage() + { + return (this.listBasePathErrorMessage); + } + + + + /******************************************************************************* + ** Setter for listBasePathErrorMessage + *******************************************************************************/ + public void setListBasePathErrorMessage(String listBasePathErrorMessage) + { + this.listBasePathErrorMessage = listBasePathErrorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for listBasePathErrorMessage + *******************************************************************************/ + public SFTPTestConnectionTestOutput withListBasePathErrorMessage(String listBasePathErrorMessage) + { + this.listBasePathErrorMessage = listBasePathErrorMessage; + return (this); + } + + + + /******************************************************************************* + ** Getter for isListBasePathSuccess + ** + *******************************************************************************/ + public Boolean getIsListBasePathSuccess() + { + return isListBasePathSuccess; + } + + + + /******************************************************************************* + ** Setter for isListBasePathSuccess + ** + *******************************************************************************/ + public void setIsListBasePathSuccess(Boolean isListBasePathSuccess) + { + this.isListBasePathSuccess = isListBasePathSuccess; + } + + + + /******************************************************************************* + ** Fluent setter for isListBasePathSuccess + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withIsListBasePathSuccess(Boolean isListBasePathSuccess) + { + this.isListBasePathSuccess = isListBasePathSuccess; + return (this); + } + + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java new file mode 100644 index 00000000..93b31a69 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateOutput execute(UpdateInput updateInput) throws QException + { + throw new NotImplementedException("SFTP update not implemented"); + /* + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List records = new ArrayList<>(); + rs.setRecords(records); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java new file mode 100644 index 00000000..06cae267 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model; + + +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** + *******************************************************************************/ +public record SFTPDirEntryWithPath(String path, SftpClient.DirEntry dirEntry) +{ +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java new file mode 100644 index 00000000..c423f99e --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java @@ -0,0 +1,198 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP backend meta data. + *******************************************************************************/ +public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData +{ + private String username; + private String password; + private String hostName; + private Integer port; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPBackendMetaData() + { + super(); + setBackendType(SFTPBackendModule.class); + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public SFTPBackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public SFTPBackendMetaData withName(String name) + { + setName(name); + return this; + } + + + + /******************************************************************************* + ** Getter for username + *******************************************************************************/ + public String getUsername() + { + return (this.username); + } + + + + /******************************************************************************* + ** Setter for username + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + *******************************************************************************/ + public SFTPBackendMetaData withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + *******************************************************************************/ + public String getPassword() + { + return (this.password); + } + + + + /******************************************************************************* + ** Setter for password + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent setter for password + *******************************************************************************/ + public SFTPBackendMetaData withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Getter for hostName + *******************************************************************************/ + public String getHostName() + { + return (this.hostName); + } + + + + /******************************************************************************* + ** Setter for hostName + *******************************************************************************/ + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + + + /******************************************************************************* + ** Fluent setter for hostName + *******************************************************************************/ + public SFTPBackendMetaData withHostName(String hostName) + { + this.hostName = hostName; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + *******************************************************************************/ + public Integer getPort() + { + return (this.port); + } + + + + /******************************************************************************* + ** Setter for port + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent setter for port + *******************************************************************************/ + public SFTPBackendMetaData withPort(Integer port) + { + this.port = port; + return (this); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java new file mode 100644 index 00000000..b5205cb9 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum SFTPBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + HOSTNAME, + PORT, + BASE_PATH +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java new file mode 100644 index 00000000..3a1605e0 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP specific Extension of QTableBackendDetails + *******************************************************************************/ +public class SFTPTableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPTableBackendDetails() + { + super(); + setBackendType(SFTPBackendModule.class); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java new file mode 100644 index 00000000..19887c2a --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java @@ -0,0 +1,179 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.utils; + + +import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.sshd.sftp.client.SftpClient; +import org.jetbrains.annotations.NotNull; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPOutputStream extends PipedOutputStream +{ + private static final QLogger LOG = QLogger.getLogger(SFTPOutputStream.class); + + private final SftpClient sftpClient; + + private final PipedInputStream pipedInputStream; + private final Future putFuture; + + private AtomicBoolean started = new AtomicBoolean(false); + private AtomicReference putException = new AtomicReference<>(null); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public SFTPOutputStream(SftpClient sftpClient, String path) throws IOException + { + pipedInputStream = new PipedInputStream(this, 32 * 1024); + + this.sftpClient = sftpClient; + + putFuture = Executors.newSingleThreadExecutor().submit(() -> + { + try + { + started.set(true); + sftpClient.put(pipedInputStream, path); + } + catch(Exception e) + { + putException.set(e); + LOG.error("Error putting SFTP output stream", e); + + try + { + pipedInputStream.close(); + } + catch(IOException ex) + { + LOG.error("Secondary error closing pipedInputStream after sftp put error", e); + } + + throw new RuntimeException(e); + } + }); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void write(@NotNull byte[] b) throws IOException + { + try + { + super.write(b); + } + catch(IOException e) + { + if(putException.get() != null) + { + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error writing to SFTP output stream", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws IOException + { + try + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // don't try to close anything until we know that the sftpClient.put call's thread // + // has tried to start (otherwise, race condition could cause us to close things too early) // + ///////////////////////////////////////////////////////////////////////////////////////////// + int sleepLoops = 0; + while(!started.get() && sleepLoops++ <= 30) + { + SleepUtils.sleep(1, TimeUnit.SECONDS); + } + + /////////////////////////////////////////////////////////////////////////////////////// + // closing the pipedOutputStream (super) causes things to flush and complete the put // + /////////////////////////////////////////////////////////////////////////////////////// + super.close(); + + //////////////////////////////// + // wait for the put to finish // + //////////////////////////////// + putFuture.get(60 - sleepLoops, TimeUnit.SECONDS); + + /////////////////////////////////////////////////////////////////////////////// + // in case the put-future never did start, throw explicitly mentioning that. // + /////////////////////////////////////////////////////////////////////////////// + if(sleepLoops >= 30) + { + throw (new Exception("future to can sftpClient.put() was never started.")); + } + } + catch(ExecutionException ee) + { + throw new IOException("Error performing SFTP put", ee); + } + catch(Exception e) + { + if(putException.get() != null) + { + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error closing SFTP output stream", e); + } + finally + { + try + { + sftpClient.close(); + } + catch(IOException e) + { + LOG.error("Error closing SFTP client", e); + } + } + } + +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index acad9100..233cad0d 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem; import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.FilesystemTableMetaDataBuilder; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; @@ -52,6 +55,10 @@ import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fil import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; import org.apache.commons.io.FileUtils; @@ -60,20 +67,26 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; - public static final String BACKEND_NAME_S3 = "s3"; - public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; - public static final String BACKEND_NAME_MOCK = "mock"; - public static final String BACKEND_NAME_MEMORY = "memory"; + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; + public static final String BACKEND_NAME_SFTP = "sftp"; + public static final String BACKEND_NAME_SFTP_WITH_VARIANTS = "sftpWithVariants"; + public static final String BACKEND_NAME_MOCK = "mock"; + public static final String BACKEND_NAME_MEMORY = "memory"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String TABLE_NAME_PERSON_SFTP = "person-sftp"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; + public static final String TABLE_NAME_SFTP_FILE = "sftp-file"; + public static final String TABLE_NAME_SFTP_FILE_VARIANTS = "sftp-file-with-variants"; + public static final String TABLE_NAME_VARIANT_OPTIONS = "variant-options-table"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter"; @@ -148,6 +161,7 @@ public class TestUtils qInstance.addBackend(defineS3Backend()); qInstance.addBackend(defineS3BackendSansPrefix()); qInstance.addTable(defineS3CSVPersonTable()); + qInstance.addTable(defineSFTPCSVPersonTable()); qInstance.addTable(defineS3BlobTable()); qInstance.addTable(defineS3BlobSansPrefixTable()); qInstance.addBackend(defineMockBackend()); @@ -155,6 +169,15 @@ public class TestUtils qInstance.addTable(defineMockPersonTable()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); + QBackendMetaData sftpBackend = defineSFTPBackend(); + qInstance.addBackend(sftpBackend); + qInstance.addTable(defineSFTPFileTable(sftpBackend)); + + QBackendMetaData sftpBackendWithVariants = defineSFTPBackendWithVariants(); + qInstance.addBackend(sftpBackendWithVariants); + qInstance.addTable(defineSFTPFileTableWithVariants(sftpBackendWithVariants)); + qInstance.addTable(defineVariantOptionsTable()); + definePersonCsvImporter(qInstance); return (qInstance); @@ -162,6 +185,21 @@ public class TestUtils + /*************************************************************************** + ** + ***************************************************************************/ + private static QTableMetaData defineVariantOptionsTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_VARIANT_OPTIONS) + .withBackendName(defineMemoryBackend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("basePath", QFieldType.STRING)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -379,6 +417,25 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineSFTPCSVPersonTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON_SFTP) + .withLabel("Person SFTP Table") + .withBackendName(BACKEND_NAME_SFTP) + .withPrimaryKeyField("id") + .withFields(defineCommonPersonTableFields()) + .withBackendDetails(new SFTPTableBackendDetails() + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) + ); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -463,4 +520,77 @@ public class TestUtils MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); return (mockAuthenticationModule.createSession(defineInstance(), null)); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTable(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackend() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + .withBasePath(BaseSFTPTest.BACKEND_FOLDER) + .withName(BACKEND_NAME_SFTP)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTableWithVariants(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE_VARIANTS) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackendWithVariants() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + + //////////////////////////////////// + // only get basePath from variant // + //////////////////////////////////// + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withOptionsTableName(TABLE_NAME_VARIANT_OPTIONS) + .withVariantTypeKey(TABLE_NAME_VARIANT_OPTIONS) + .withBackendSettingSourceFieldNameMap(Map.of( + SFTPBackendVariantSetting.BASE_PATH, "basePath" + )) + ) + .withName(BACKEND_NAME_SFTP_WITH_VARIANTS)); + } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java new file mode 100644 index 00000000..42c968be --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java @@ -0,0 +1,154 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.module.filesystem.BaseTest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.MountableFile; + + +/******************************************************************************* + ** Base class for tests that want to be able to work with sftp testcontainer + *******************************************************************************/ +public class BaseSFTPTest extends BaseTest +{ + public static final int PORT = 22; + public static final String USERNAME = "testuser"; + public static final String PASSWORD = "testpass"; + public static final String HOST_NAME = "localhost"; + + public static final String BACKEND_FOLDER = "upload"; + public static final String TABLE_FOLDER = "files"; + public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER; + + private static GenericContainer sftpContainer; + private static Integer currentPort; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @BeforeAll + static void setUp() throws Exception + { + sftpContainer = new GenericContainer<>("atmoz/sftp:latest") + .withExposedPorts(PORT) + .withCommand(USERNAME + ":" + PASSWORD + ":1001"); + + sftpContainer.start(); + + for(int i = 0; i < 5; i++) + { + copyFileToContainer("files/testfile.txt", REMOTE_DIR + "/testfile-" + i + ".txt"); + } + + grantUploadFilesDirWritePermission(); + + currentPort = sftpContainer.getMappedPort(22); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void copyFileToContainer(String sourceFileClasspathResourceName, String fullRemotePath) + { + sftpContainer.copyFileToContainer(MountableFile.forClasspathResource(sourceFileClasspathResourceName), fullRemotePath); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void rmrfInContainer(String fullRemotePath) throws Exception + { + sftpContainer.execInContainer("rm", "-rf", fullRemotePath); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + @AfterAll + static void tearDown() + { + if(sftpContainer != null) + { + sftpContainer.stop(); + } + } + + + + /******************************************************************************* + ** Getter for currentPort + ** + *******************************************************************************/ + public static Integer getCurrentPort() + { + return currentPort; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void revokeUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("444"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void grantUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("777"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void setUploadFilesDirPermission(String mode) throws Exception + { + sftpContainer.execInContainer("chmod", mode, "/home/testuser/upload/files"); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception + { + sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path); + } +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java new file mode 100644 index 00000000..643fe2ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java @@ -0,0 +1,64 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCount1() throws QException + { + CountInput countInput = initCountRequest(); + SFTPCountAction countAction = new SFTPCountAction(); + CountOutput countOutput = countAction.execute(countInput); + Assertions.assertEquals(5, countOutput.getCount(), "Expected # of rows from unfiltered count"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private CountInput initCountRequest() throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + return countInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java new file mode 100644 index 00000000..3a992fbf --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + assertThrows(NotImplementedException.class, () -> new SFTPDeleteAction().execute(new DeleteInput())); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java new file mode 100644 index 00000000..c648021c --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java @@ -0,0 +1,120 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.IOException; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOne() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + assertThat(insertOutput.getRecords()) + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseSFTPTest.BACKEND_FOLDER)); + + QRecord record = insertOutput.getRecords().get(0); + String fullPath = record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH); + assertThat(record.getErrors()).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOnePermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + + QRecord record = insertOutput.getRecords().get(0); + assertThat(record.getErrors()).isNotEmpty(); + assertThat(record.getErrors().get(0).getMessage()).contains("Error writing file: Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityMany() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_SFTP); + insertInput.setRecords(List.of( + new QRecord().withValue("id", "1").withValue("firstName", "Bob") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + assertThatThrownBy(() -> insertAction.execute(insertInput)) + .hasRootCauseInstanceOf(NotImplementedException.class); + } +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java new file mode 100644 index 00000000..9708f68e --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -0,0 +1,162 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Unit test for SFTPQueryAction + *******************************************************************************/ +class SFTPQueryActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSimpleQuery() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueryWithPath() throws Exception + { + String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/"; + try + { + copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt"); + copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt"); + + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE) + .withFilter(new QQueryFilter(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows from subfolder path query"); + } + finally + { + rmrfInContainer(subfolderPath); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueryWithPathAndNameLike() throws Exception + { + String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/"; + try + { + copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt"); + copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt"); + copyFileToContainer("files/testfile.txt", subfolderPath + "/who.txt"); + + Map patternExpectedCountMap = Map.of( + "%.txt", 3, + "sub%", 2, + "%1%", 1, + "%", 3, + "*", 0 + ); + + for(Map.Entry entry : patternExpectedCountMap.entrySet()) + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")) + .withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey()))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey()); + } + } + finally + { + rmrfInContainer(subfolderPath); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQueryVariantsTable() throws Exception + { + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_VARIANT_OPTIONS).withRecords(List.of( + new QRecord().withValue("id", 1).withValue("basePath", BaseSFTPTest.BACKEND_FOLDER), + new QRecord().withValue("id", 2).withValue("basePath", "empty-folder"), + new QRecord().withValue("id", 3).withValue("basePath", "non-existing-path") + ))); + + mkdirInSftpContainerUnderHomeTestuser("empty-folder/files"); + + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'variant-options-table' for Backend"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 2)); + queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(0, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 3)); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .rootCause() + .hasMessageContaining("No such file"); + + // Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java new file mode 100644 index 00000000..ed49cc2d --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FilesystemStorageAction + *******************************************************************************/ +public class SFTPStorageActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSmall() throws Exception + { + String data = "Hellooo, Storage."; + runTest(data); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testPermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + String data = "oops!"; + assertThatThrownBy(() -> runTest(data)) + .hasRootCauseInstanceOf(IOException.class) + .rootCause() + .hasMessageContaining("Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLarge() throws Exception + { + String data = StringUtils.join("!", Collections.nCopies(5_000_000, "Hello")); + runTest(data); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void runTest(String data) throws QException, IOException + { + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_SFTP_FILE).withReference("fromStorageAction.txt"); + + StorageAction storageAction = new StorageAction(); + OutputStream outputStream = storageAction.createOutputStream(storageInput); + outputStream.write(data.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + InputStream inputStream = storageAction.getInputStream(storageInput); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + inputStream.transferTo(byteArrayOutputStream); + + assertEquals(data.length(), byteArrayOutputStream.toString(StandardCharsets.UTF_8).length()); + assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8)); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java new file mode 100644 index 00000000..b1439cbf --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java @@ -0,0 +1,178 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for SFTPTestConnectionAction + *******************************************************************************/ +class SFTPTestConnectionActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessWithoutPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessWithPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME) + .withBasePath(BaseSFTPTest.BACKEND_FOLDER); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertTrue(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessfulConnectFailedPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME) + .withBasePath("no-such-path"); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertFalse(output.getIsListBasePathSuccess()); + assertNotNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadUsername() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername("not-" + BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadPassword() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword("not-" + BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadHostname() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName("not-" + BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadPort() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(10 * BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java new file mode 100644 index 00000000..2f025320 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + assertThrows(NotImplementedException.class, () -> new SFTPUpdateAction().execute(new UpdateInput())); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt new file mode 100644 index 00000000..eab448b1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt @@ -0,0 +1,3 @@ +This is a file. + +It is a test. diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index d6b89246..d9749d86 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -132,6 +133,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(QRecord record : page) { QRecord outputRecord = new QRecord(record); + if(!StringUtils.hasContent(outputRecord.getTableName())) + { + outputRecord.setTableName(tableName); + } outputRecords.add(outputRecord); } continue; @@ -151,6 +156,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(QRecord record : page) { QRecord outputRecord = new QRecord(record); + if(!StringUtils.hasContent(outputRecord.getTableName())) + { + outputRecord.setTableName(tableName); + } outputRecords.add(outputRecord); if(CollectionUtils.nullSafeIsEmpty(record.getErrors())) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index c82274d6..80f227d3 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -95,8 +95,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; @@ -1200,15 +1198,18 @@ public class QJavalinImplementation ///////////////////////////////////////////////////////////////////////////////////// if(backend != null && backend.getUsesVariants()) { - queryInput.setTableName(backend.getVariantOptionsTableName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backend.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backend.getVariantOptionsTableTypeValue()))); + QTableMetaData variantsTable = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName()); + + queryInput.setTableName(variantsTable.getName()); + queryInput.setFilter(backend.getBackendVariantsConfig().getOptionsFilter()); + queryInput.setShouldGenerateDisplayValues(true); QueryOutput output = new QueryAction().execute(queryInput); for(QRecord qRecord : output.getRecords()) { variants.add(new QFrontendVariant() - .withId(qRecord.getValue(backend.getVariantOptionsTableIdField())) - .withType(backend.getVariantOptionsTableTypeValue()) - .withName(qRecord.getValueString(backend.getVariantOptionsTableNameField()))); + .withId(qRecord.getValue(variantsTable.getPrimaryKeyField())) + .withType(backend.getBackendVariantsConfig().getVariantTypeKey()) + .withName(qRecord.getRecordLabel())); } QJavalinAccessLogger.logStartSilent("variants");