QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged feature/sftp-and-headless-bulk-load into integration/sprint-61

Browse Source
This commit is contained in:
darin.kelkhoff
2025-02-20 09:30:07 -06:00
parent ead66385be be4f3c68f0
commit b1f3613e00
71 changed files with 5189 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java
View File

@ -815,13 +815,14 @@ public class RunProcessAction
{
QSession session = QContext.getQSession();
QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend());
if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey))
{
LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
}
else
{
basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
basepullKeyValue += "-" + session.getBackendVariants().get(variantTypeKey);
}
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java
View File

@ -82,6 +82,11 @@ public class DeleteAction
{
ActionHelper.validateSession(deleteInput);
if(deleteInput.getTableName() == null)
{
throw (new QException("Table name was not specified in delete input"));
}
QTableMetaData table = deleteInput.getTable();
String primaryKeyFieldName = table.getPrimaryKeyField();
QFieldMetaData primaryKeyField = table.getField(primaryKeyFieldName);

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -110,6 +111,12 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
public InsertOutput execute(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);
if(!StringUtils.hasContent(insertInput.getTableName()))
{
throw (new QException("Table name was not specified in update input"));
}
QTableMetaData table = insertInput.getTable();
if(table == null)

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java
View File

@ -47,7 +47,8 @@ public class StorageAction
{
/*******************************************************************************
**
** create an output stream in the storage backend - that can be written to,
** for the purpose of inserting or writing a file into storage.
*******************************************************************************/
public OutputStream createOutputStream(StorageInput storageInput) throws QException
{
@ -59,7 +60,8 @@ public class StorageAction
/*******************************************************************************
**
** create an input stream in the storage backend - that can be read from,
** for the purpose of getting or reading a file from storage.
*******************************************************************************/
public InputStream getInputStream(StorageInput storageInput) throws QException
{

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java
View File

@ -74,6 +74,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang.BooleanUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -118,6 +119,11 @@ public class UpdateAction
{
ActionHelper.validateSession(updateInput);
if(!StringUtils.hasContent(updateInput.getTableName()))
{
throw (new QException("Table name was not specified in update input"));
}
QTableMetaData table = updateInput.getTable();
//////////////////////////////////////////////////////

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
View File

@ -23,14 +23,18 @@ package com.kingsrook.qqq.backend.core.actions.values;
import java.io.Serializable;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
@ -508,7 +512,7 @@ public class QValueFormatter
{
@SuppressWarnings("unchecked") // instance validation should make this safe!
List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
List<String> values = CollectionUtils.nullSafeHasContents(fileNameFormatFields) ? fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList() : Collections.emptyList();
fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
}
}
@ -531,7 +535,7 @@ public class QValueFormatter
////////////////////////////////////////////////////////////////////////////////////////////////
// if field type is blob OR if there's a supplemental process or code-ref that needs to run - //
// then update its value to be a callback-url that'll give access to the bytes to download //
// then update its value to be a callback-url that'll give access to the bytes to download. //
// implied here is that a String value (w/o supplemental code/proc) has its value stay as a //
// URL, which is where the file is directly downloaded from. And in the case of a String //
// with code-to-run, then the code should run, followed by a redirect to the value URL. //
@ -540,7 +544,10 @@ public class QValueFormatter
|| adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE)
|| adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME))
{
record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
record.setValue(field.getName(), "/data/" + table.getName() + "/"
+ URLEncoder.encode(ValueUtils.getValueAsString(primaryKey), StandardCharsets.UTF_8) + "/"
+ field.getName() + "/"
+ URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8));
}
record.setDisplayValue(field.getName(), fileName);
}

98
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -37,7 +37,9 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TimeZone;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
@ -108,12 +110,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.Automatio
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ListingHash;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda;
import org.apache.commons.lang.BooleanUtils;
import org.quartz.CronExpression;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -543,6 +549,60 @@ public class QInstanceValidator
{
assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
///////////////////////
// validate variants //
///////////////////////
BackendVariantsConfig backendVariantsConfig = backend.getBackendVariantsConfig();
if(BooleanUtils.isTrue(backend.getUsesVariants()))
{
if(assertCondition(backendVariantsConfig != null, "Missing backendVariantsConfig in backend [" + backendName + "] which is marked as usesVariants"))
{
assertCondition(StringUtils.hasContent(backendVariantsConfig.getVariantTypeKey()), "Missing variantTypeKey in backendVariantsConfig in [" + backendName + "]");
String optionsTableName = backendVariantsConfig.getOptionsTableName();
QTableMetaData optionsTable = qInstance.getTable(optionsTableName);
if(assertCondition(StringUtils.hasContent(optionsTableName), "Missing optionsTableName in backendVariantsConfig in [" + backendName + "]"))
{
if(assertCondition(optionsTable != null, "Unrecognized optionsTableName [" + optionsTableName + "] in backendVariantsConfig in [" + backendName + "]"))
{
QQueryFilter optionsFilter = backendVariantsConfig.getOptionsFilter();
if(optionsFilter != null)
{
validateQueryFilter(qInstance, "optionsFilter in backendVariantsConfig in backend [" + backendName + "]: ", optionsTable, optionsFilter, null);
}
}
}
Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap = backendVariantsConfig.getBackendSettingSourceFieldNameMap();
if(assertCondition(CollectionUtils.nullSafeHasContents(backendSettingSourceFieldNameMap), "Missing or empty backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]"))
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
// only validate field names in the backendSettingSourceFieldNameMap if there is NOT a variantRecordSupplier //
// (the idea being, that the supplier might be building a record with fieldNames that aren't in the table... //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(optionsTable != null && backendVariantsConfig.getVariantRecordLookupFunction() == null)
{
for(Map.Entry<BackendVariantSetting, String> entry : backendSettingSourceFieldNameMap.entrySet())
{
assertCondition(optionsTable.getFields().containsKey(entry.getValue()), "Unrecognized fieldName [" + entry.getValue() + "] in backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]");
}
}
}
if(backendVariantsConfig.getVariantRecordLookupFunction() != null)
{
validateSimpleCodeReference("VariantRecordSupplier in backendVariantsConfig in backend [" + backendName + "]: ", backendVariantsConfig.getVariantRecordLookupFunction(), UnsafeFunction.class, Function.class);
}
}
}
else
{
assertCondition(backendVariantsConfig == null, "Should not have a backendVariantsConfig in backend [" + backendName + "] which is not marked as usesVariants");
}
///////////////////////////////////////////
// let the backend do its own validation //
///////////////////////////////////////////
backend.performValidation(this);
runPlugins(QBackendMetaData.class, backend, qInstance);
@ -1356,7 +1416,7 @@ public class QInstanceValidator
////////////////////////////////////////////////////////////////////////
if(customizerInstance != null && tableCustomizer.getExpectedType() != null)
{
assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance);
assertObjectCanBeCasted(prefix, customizerInstance, tableCustomizer.getExpectedType());
}
}
}
@ -1368,18 +1428,31 @@ public class QInstanceValidator
/*******************************************************************************
** Make sure that a given object can be casted to an expected type.
*******************************************************************************/
private <T> T assertObjectCanBeCasted(String errorPrefix, Class<T> expectedType, Object object)
private void assertObjectCanBeCasted(String errorPrefix, Object object, Class<?>... anyOfExpectedClasses)
{
T castedObject = null;
try
for(Class<?> expectedClass : anyOfExpectedClasses)
{
castedObject = expectedType.cast(object);
try
{
expectedClass.cast(object);
return;
}
catch(ClassCastException e)
{
/////////////////////////////////////
// try next type (if there is one) //
/////////////////////////////////////
}
}
catch(ClassCastException e)
if(anyOfExpectedClasses.length == 1)
{
errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType);
errors.add(errorPrefix + "CodeReference is not of the expected type: " + anyOfExpectedClasses[0]);
}
else
{
errors.add(errorPrefix + "CodeReference is not any of the expected types: " + Arrays.stream(anyOfExpectedClasses).map(c -> c.getName()).collect(Collectors.joining(", ")));
}
return castedObject;
}
@ -1616,12 +1689,12 @@ public class QInstanceValidator
for(QFieldMetaData field : process.getInputFields())
{
validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName());
validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName() + " ");
}
for(QFieldMetaData field : process.getOutputFields())
{
validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName());
validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName() + " ");
}
if(process.getCancelStep() != null)
@ -2123,7 +2196,8 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class<?> expectedClass)
@SafeVarargs
private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class<?>... anyOfExpectedClasses)
{
if(!preAssertionsForCodeReference(codeReference, prefix))
{
@ -2151,7 +2225,7 @@ public class QInstanceValidator
////////////////////////////////////////////////////////////////////////
if(classInstance != null)
{
assertObjectCanBeCasted(prefix, expectedClass, classInstance);
assertObjectCanBeCasted(prefix, classInstance, anyOfExpectedClasses);
}
}
}

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java
View File

@ -326,6 +326,20 @@ public class AuditSingleInput implements Serializable
/*******************************************************************************
** Fluent setter for details
*******************************************************************************/
public AuditSingleInput withDetailMessages(List<String> details)
{
for(String detail : details)
{
addDetail(message);
}
return (this);
}
/*******************************************************************************
**
*******************************************************************************/

25
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
import java.io.Serializable;
import java.time.Instant;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -34,9 +35,11 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback;
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface;
@ -247,6 +250,26 @@ public class RunBackendStepInput extends AbstractActionInput
/*******************************************************************************
** Getter for records converted to entities of a given type.
**
*******************************************************************************/
public <E extends QRecordEntity> List<E> getRecordsAsEntities(Class<E> entityClass) throws QException
{
List<E> rs = new ArrayList<>();
///////////////////////////////////////////////////////////////////////////////////
// note - important to call getRecords here, which is overwritten in subclasses! //
///////////////////////////////////////////////////////////////////////////////////
for(QRecord record : getRecords())
{
rs.add(QRecordEntity.fromQRecord(entityClass, record));
}
return (rs);
}
/*******************************************************************************
** Setter for records
**
@ -582,7 +605,7 @@ public class RunBackendStepInput extends AbstractActionInput
***************************************************************************/
public void traceMessage(ProcessTracerMessage message)
{
if(processTracer != null)
if(processTracer != null && message != null)
{
try
{

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java
View File

@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -258,7 +259,7 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
/*******************************************************************************
**
** add a record to the step output, e.g., for going through to the next step.
*******************************************************************************/
public void addRecord(QRecord record)
{
@ -271,6 +272,16 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
/***************************************************************************
** add a RecordEntity to the step output, e.g., for going through to the next step.
***************************************************************************/
public void addRecordEntity(QRecordEntity recordEntity)
{
addRecord(recordEntity.toQRecord());
}
/*******************************************************************************
** Getter for auditInputList
*******************************************************************************/

47
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java Normal file
View File

@ -0,0 +1,47 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata;
import com.kingsrook.qqq.backend.core.logging.QLogger;
/*******************************************************************************
** for use-cases where a metaDataProducer directly adds its objects to the
** qInstance, then this empty object can be returned.
*******************************************************************************/
public class EmptyMetaDataProducerOutput implements MetaDataProducerOutput
{
private static final QLogger LOG = QLogger.getLogger(EmptyMetaDataProducerOutput.class);
/***************************************************************************
**
***************************************************************************/
@Override
public void addSelfToInstance(QInstance instance)
{
/////////////////////////////////
// noop - this output is empty //
/////////////////////////////////
LOG.trace("empty meta data producer has nothing to add.");
}
}

235
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java
View File

@ -26,8 +26,13 @@ import java.util.HashSet;
import java.util.Set;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.serialization.QBackendMetaDataDeserializer;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@ -45,21 +50,18 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
private Set<Capability> enabledCapabilities = new HashSet<>();
private Set<Capability> disabledCapabilities = new HashSet<>();
private Boolean usesVariants = false;
private String variantOptionsTableIdField;
private String variantOptionsTableNameField;
private String variantOptionsTableTypeField;
private String variantOptionsTableTypeValue;
private String variantOptionsTableUsernameField;
private String variantOptionsTablePasswordField;
private String variantOptionsTableApiKeyField;
private String variantOptionsTableClientIdField;
private String variantOptionsTableClientSecretField;
private String variantOptionsTableName;
private Boolean usesVariants = false;
private BackendVariantsConfig backendVariantsConfig;
// todo - at some point, we may want to apply this to secret properties on subclasses?
// @JsonFilter("secretsFilter")
@Deprecated(since = "Replaced by filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
private String variantOptionsTableTypeField; // a field on which to filter the variant-options table, to limit which records in it are available as variants
@Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
private String variantOptionsTableTypeValue; // value for the type-field, to limit which records in it are available as variants; but also, the key in the session.backendVariants map!
/*******************************************************************************
@ -394,22 +396,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Getter for variantOptionsTableIdField
*******************************************************************************/
public String getVariantOptionsTableIdField()
{
return (this.variantOptionsTableIdField);
}
/*******************************************************************************
** Setter for variantOptionsTableIdField
*******************************************************************************/
@Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key")
public void setVariantOptionsTableIdField(String variantOptionsTableIdField)
{
this.variantOptionsTableIdField = variantOptionsTableIdField;
/////////////////////////////////////////////////
// noop as we migrate to backendVariantsConfig //
/////////////////////////////////////////////////
}
@ -417,30 +412,24 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableIdField
*******************************************************************************/
@Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key")
public QBackendMetaData withVariantOptionsTableIdField(String variantOptionsTableIdField)
{
this.variantOptionsTableIdField = variantOptionsTableIdField;
this.setVariantOptionsTableIdField(variantOptionsTableIdField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableNameField
*******************************************************************************/
public String getVariantOptionsTableNameField()
{
return (this.variantOptionsTableNameField);
}
/*******************************************************************************
** Setter for variantOptionsTableNameField
*******************************************************************************/
@Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel")
public void setVariantOptionsTableNameField(String variantOptionsTableNameField)
{
this.variantOptionsTableNameField = variantOptionsTableNameField;
/////////////////////////////////////////////////
// noop as we migrate to backendVariantsConfig //
/////////////////////////////////////////////////
}
@ -448,30 +437,26 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableNameField
*******************************************************************************/
@Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel")
public QBackendMetaData withVariantOptionsTableNameField(String variantOptionsTableNameField)
{
this.variantOptionsTableNameField = variantOptionsTableNameField;
this.setVariantOptionsTableNameField(variantOptionsTableNameField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableTypeField
*******************************************************************************/
public String getVariantOptionsTableTypeField()
{
return (this.variantOptionsTableTypeField);
}
/*******************************************************************************
** Setter for variantOptionsTableTypeField
*******************************************************************************/
@Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
public void setVariantOptionsTableTypeField(String variantOptionsTableTypeField)
{
this.variantOptionsTableTypeField = variantOptionsTableTypeField;
if(this.variantOptionsTableTypeValue != null)
{
this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue)));
}
}
@ -479,30 +464,28 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableTypeField
*******************************************************************************/
@Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
public QBackendMetaData withVariantOptionsTableTypeField(String variantOptionsTableTypeField)
{
this.variantOptionsTableTypeField = variantOptionsTableTypeField;
this.setVariantOptionsTableTypeField(variantOptionsTableTypeField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableTypeValue
*******************************************************************************/
public String getVariantOptionsTableTypeValue()
{
return (this.variantOptionsTableTypeValue);
}
/*******************************************************************************
** Setter for variantOptionsTableTypeValue
*******************************************************************************/
@Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
public void setVariantOptionsTableTypeValue(String variantOptionsTableTypeValue)
{
this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeValue);
this.variantOptionsTableTypeValue = variantOptionsTableTypeValue;
if(this.variantOptionsTableTypeField != null)
{
this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue)));
}
}
@ -510,30 +493,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableTypeValue
*******************************************************************************/
@Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
public QBackendMetaData withVariantOptionsTableTypeValue(String variantOptionsTableTypeValue)
{
this.variantOptionsTableTypeValue = variantOptionsTableTypeValue;
this.setVariantOptionsTableTypeValue(variantOptionsTableTypeValue);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableUsernameField
*******************************************************************************/
public String getVariantOptionsTableUsernameField()
{
return (this.variantOptionsTableUsernameField);
}
/*******************************************************************************
** Setter for variantOptionsTableUsernameField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTableUsernameField(String variantOptionsTableUsernameField)
{
this.variantOptionsTableUsernameField = variantOptionsTableUsernameField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.USERNAME, variantOptionsTableUsernameField);
}
@ -541,30 +516,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableUsernameField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTableUsernameField(String variantOptionsTableUsernameField)
{
this.variantOptionsTableUsernameField = variantOptionsTableUsernameField;
this.setVariantOptionsTableUsernameField(variantOptionsTableUsernameField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTablePasswordField
*******************************************************************************/
public String getVariantOptionsTablePasswordField()
{
return (this.variantOptionsTablePasswordField);
}
/*******************************************************************************
** Setter for variantOptionsTablePasswordField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTablePasswordField(String variantOptionsTablePasswordField)
{
this.variantOptionsTablePasswordField = variantOptionsTablePasswordField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.PASSWORD, variantOptionsTablePasswordField);
}
@ -572,30 +539,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTablePasswordField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTablePasswordField(String variantOptionsTablePasswordField)
{
this.variantOptionsTablePasswordField = variantOptionsTablePasswordField;
this.setVariantOptionsTablePasswordField(variantOptionsTablePasswordField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableApiKeyField
*******************************************************************************/
public String getVariantOptionsTableApiKeyField()
{
return (this.variantOptionsTableApiKeyField);
}
/*******************************************************************************
** Setter for variantOptionsTableApiKeyField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField)
{
this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.API_KEY, variantOptionsTableApiKeyField);
}
@ -603,30 +562,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableApiKeyField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField)
{
this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField;
this.setVariantOptionsTableApiKeyField(variantOptionsTableApiKeyField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableName
*******************************************************************************/
public String getVariantOptionsTableName()
{
return (this.variantOptionsTableName);
}
/*******************************************************************************
** Setter for variantOptionsTableName
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.tableName")
public void setVariantOptionsTableName(String variantOptionsTableName)
{
this.variantOptionsTableName = variantOptionsTableName;
this.getOrWithNewBackendVariantsConfig().withOptionsTableName(variantOptionsTableName);
}
@ -634,9 +585,10 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableName
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.tableName")
public QBackendMetaData withVariantOptionsTableName(String variantOptionsTableName)
{
this.variantOptionsTableName = variantOptionsTableName;
this.setVariantOptionsTableName(variantOptionsTableName);
return (this);
}
@ -651,22 +603,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
qInstance.addBackend(this);
}
/*******************************************************************************
** Getter for variantOptionsTableClientIdField
*******************************************************************************/
public String getVariantOptionsTableClientIdField()
{
return (this.variantOptionsTableClientIdField);
}
/*******************************************************************************
** Setter for variantOptionsTableClientIdField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
{
this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_ID, variantOptionsTableClientIdField);
}
@ -674,30 +619,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableClientIdField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
{
this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
this.setVariantOptionsTableClientIdField(variantOptionsTableClientIdField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableClientSecretField
*******************************************************************************/
public String getVariantOptionsTableClientSecretField()
{
return (this.variantOptionsTableClientSecretField);
}
/*******************************************************************************
** Setter for variantOptionsTableClientSecretField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
{
this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_SECRET, variantOptionsTableClientSecretField);
}
@ -705,11 +642,55 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableClientSecretField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
{
this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
this.setVariantOptionsTableClientSecretField(variantOptionsTableClientSecretField);
return (this);
}
/*******************************************************************************
** Getter for backendVariantsConfig
*******************************************************************************/
public BackendVariantsConfig getBackendVariantsConfig()
{
return (this.backendVariantsConfig);
}
/*******************************************************************************
** Setter for backendVariantsConfig
*******************************************************************************/
public void setBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig)
{
this.backendVariantsConfig = backendVariantsConfig;
}
/*******************************************************************************
** Fluent setter for backendVariantsConfig
*******************************************************************************/
public QBackendMetaData withBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig)
{
this.backendVariantsConfig = backendVariantsConfig;
return (this);
}
/***************************************************************************
**
***************************************************************************/
private BackendVariantsConfig getOrWithNewBackendVariantsConfig()
{
if(backendVariantsConfig == null)
{
setBackendVariantsConfig(new BackendVariantsConfig());
}
return backendVariantsConfig;
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java
View File

@ -30,4 +30,5 @@ public enum AuditLevel
NONE,
RECORD,
FIELD
// idea: only audit changes to fields, e.g., on edit. though, is that a different dimension than this?
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java
View File

@ -86,7 +86,6 @@ public class QFrontendTableMetaData
//////////////////////////////////////////////////////////////////////////////////
/*******************************************************************************
**
*******************************************************************************/
@ -170,7 +169,7 @@ public class QFrontendTableMetaData
if(backend != null && backend.getUsesVariants())
{
usesVariants = true;
variantTableLabel = QContext.getQInstance().getTable(backend.getVariantOptionsTableName()).getLabel();
variantTableLabel = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName()).getLabel();
}
this.helpContents = tableMetaData.getHelpContent();

20
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java
View File

@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock
import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -712,6 +713,25 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
/*******************************************************************************
** Getter for sections
**
*******************************************************************************/
public QFieldSection getSection(String name)
{
for(QFieldSection qFieldSection : CollectionUtils.nonNullList(sections))
{
if(qFieldSection.getName().equals(name))
{
return (qFieldSection);
}
}
return (null);
}
/*******************************************************************************
** Setter for sections
**

221
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java Normal file
View File

@ -0,0 +1,221 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
/*******************************************************************************
** Factory class for creating "standard" qfield sections. e.g., if you want
** the same t1, t2, and t3 section on all your tables, use this class to
** produce them.
**
** You can change the default name & iconNames for those sections, but note,
** this is a static/utility style class, so those settings are static fields.
**
** The method customT2 is provided as not much of a shortcut over "doing it yourself",
** but to allow all sections for a table to be produced through calls to this factory,
** so they look more similar.
*******************************************************************************/
public class SectionFactory
{
private static String defaultT1name = "identity";
private static String defaultT1iconName = "badge";
private static String defaultT2name = "data";
private static String defaultT2iconName = "text_snippet";
private static String defaultT3name = "dates";
private static String defaultT3iconName = "calendar_month";
/*******************************************************************************
** private constructor, to enforce static usage, e.g., to make clear the fields
** are static fields.
**
*******************************************************************************/
private SectionFactory()
{
}
/***************************************************************************
**
***************************************************************************/
public static QFieldSection defaultT1(String... fieldNames)
{
return new QFieldSection(defaultT1name, new QIcon().withName(defaultT1iconName), Tier.T1, List.of(fieldNames));
}
/***************************************************************************
**
***************************************************************************/
public static QFieldSection defaultT2(String... fieldNames)
{
return new QFieldSection(defaultT2name, new QIcon().withName(defaultT2iconName), Tier.T2, List.of(fieldNames));
}
/***************************************************************************
**
***************************************************************************/
public static QFieldSection customT2(String name, QIcon icon, String... fieldNames)
{
return new QFieldSection(name, icon, Tier.T2, List.of(fieldNames));
}
/***************************************************************************
**
***************************************************************************/
public static QFieldSection defaultT3(String... fieldNames)
{
return new QFieldSection(defaultT3name, new QIcon().withName(defaultT3iconName), Tier.T3, List.of(fieldNames));
}
/*******************************************************************************
** Getter for defaultT1name
*******************************************************************************/
public static String getDefaultT1name()
{
return (SectionFactory.defaultT1name);
}
/*******************************************************************************
** Setter for defaultT1name
*******************************************************************************/
public static void setDefaultT1name(String defaultT1name)
{
SectionFactory.defaultT1name = defaultT1name;
}
/*******************************************************************************
** Getter for defaultT1iconName
*******************************************************************************/
public static String getDefaultT1iconName()
{
return (SectionFactory.defaultT1iconName);
}
/*******************************************************************************
** Setter for defaultT1iconName
*******************************************************************************/
public static void setDefaultT1iconName(String defaultT1iconName)
{
SectionFactory.defaultT1iconName = defaultT1iconName;
}
/*******************************************************************************
** Getter for defaultT2name
*******************************************************************************/
public static String getDefaultT2name()
{
return (SectionFactory.defaultT2name);
}
/*******************************************************************************
** Setter for defaultT2name
*******************************************************************************/
public static void setDefaultT2name(String defaultT2name)
{
SectionFactory.defaultT2name = defaultT2name;
}
/*******************************************************************************
** Getter for defaultT2iconName
*******************************************************************************/
public static String getDefaultT2iconName()
{
return (SectionFactory.defaultT2iconName);
}
/*******************************************************************************
** Setter for defaultT2iconName
*******************************************************************************/
public static void setDefaultT2iconName(String defaultT2iconName)
{
SectionFactory.defaultT2iconName = defaultT2iconName;
}
/*******************************************************************************
** Getter for defaultT3name
*******************************************************************************/
public static String getDefaultT3name()
{
return (SectionFactory.defaultT3name);
}
/*******************************************************************************
** Setter for defaultT3name
*******************************************************************************/
public static void setDefaultT3name(String defaultT3name)
{
SectionFactory.defaultT3name = defaultT3name;
}
/*******************************************************************************
** Getter for defaultT3iconName
*******************************************************************************/
public static String getDefaultT3iconName()
{
return (SectionFactory.defaultT3iconName);
}
/*******************************************************************************
** Setter for defaultT3iconName
*******************************************************************************/
public static void setDefaultT3iconName(String defaultT3iconName)
{
SectionFactory.defaultT3iconName = defaultT3iconName;
}
}

31
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java Normal file
View File

@ -0,0 +1,31 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
/*******************************************************************************
** interface to be implemented by enums (presumably) that define the possible
** settings a particular backend type can get from a variant record.
*******************************************************************************/
public interface BackendVariantSetting
{
}

225
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java Normal file
View File

@ -0,0 +1,225 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
import java.util.HashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
/*******************************************************************************
** Configs for how a backend that uses variants works. Specifically:
**
** - the variant "type key" - e.g., key for variants map in session.
** - what table supplies the variant options (optionsTableName
** - an optional filter to apply to that options table
** - a map of the settings that a backend gets from its variant table to the
** field names in that table that they come from. e.g., a backend may have a
** username attribute, whose value comes from a field named "theUser" in the
** variant options table.
** - an optional code reference to a variantRecordLookupFunction - to customize
** how the variant record is looked up (such as, adding joined or other custom
** fields).
*******************************************************************************/
public class BackendVariantsConfig
{
private String variantTypeKey;
private String optionsTableName;
private QQueryFilter optionsFilter;
private QCodeReference variantRecordLookupFunction;
private Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap;
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getOptionsTableName()
{
return (this.optionsTableName);
}
/*******************************************************************************
** Setter for tableName
*******************************************************************************/
public void setOptionsTableName(String optionsTableName)
{
this.optionsTableName = optionsTableName;
}
/*******************************************************************************
** Getter for filter
*******************************************************************************/
public QQueryFilter getOptionsFilter()
{
return (this.optionsFilter);
}
/*******************************************************************************
** Setter for filter
*******************************************************************************/
public void setOptionsFilter(QQueryFilter optionsFilter)
{
this.optionsFilter = optionsFilter;
}
/*******************************************************************************
** Getter for backendSettingSourceFieldNameMap
*******************************************************************************/
public Map<BackendVariantSetting, String> getBackendSettingSourceFieldNameMap()
{
return (this.backendSettingSourceFieldNameMap);
}
/*******************************************************************************
** Setter for backendSettingSourceFieldNameMap
*******************************************************************************/
public void setBackendSettingSourceFieldNameMap(Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap)
{
this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap;
}
/*******************************************************************************
** Fluent setter for backendSettingSourceFieldNameMap
*******************************************************************************/
public BackendVariantsConfig withBackendSettingSourceFieldName(BackendVariantSetting backendVariantSetting, String sourceFieldName)
{
if(this.backendSettingSourceFieldNameMap == null)
{
this.backendSettingSourceFieldNameMap = new HashMap<>();
}
this.backendSettingSourceFieldNameMap.put(backendVariantSetting, sourceFieldName);
return (this);
}
/*******************************************************************************
** Fluent setter for backendSettingSourceFieldNameMap
*******************************************************************************/
public BackendVariantsConfig withBackendSettingSourceFieldNameMap(Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap)
{
this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap;
return (this);
}
/*******************************************************************************
** Getter for variantTypeKey
*******************************************************************************/
public String getVariantTypeKey()
{
return (this.variantTypeKey);
}
/*******************************************************************************
** Setter for variantTypeKey
*******************************************************************************/
public void setVariantTypeKey(String variantTypeKey)
{
this.variantTypeKey = variantTypeKey;
}
/*******************************************************************************
** Fluent setter for variantTypeKey
*******************************************************************************/
public BackendVariantsConfig withVariantTypeKey(String variantTypeKey)
{
this.variantTypeKey = variantTypeKey;
return (this);
}
/*******************************************************************************
** Fluent setter for optionsTableName
*******************************************************************************/
public BackendVariantsConfig withOptionsTableName(String optionsTableName)
{
this.optionsTableName = optionsTableName;
return (this);
}
/*******************************************************************************
** Fluent setter for optionsFilter
*******************************************************************************/
public BackendVariantsConfig withOptionsFilter(QQueryFilter optionsFilter)
{
this.optionsFilter = optionsFilter;
return (this);
}
/*******************************************************************************
** Getter for variantRecordLookupFunction
*******************************************************************************/
public QCodeReference getVariantRecordLookupFunction()
{
return (this.variantRecordLookupFunction);
}
/*******************************************************************************
** Setter for variantRecordLookupFunction
*******************************************************************************/
public void setVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction)
{
this.variantRecordLookupFunction = variantRecordLookupFunction;
}
/*******************************************************************************
** Fluent setter for variantRecordLookupFunction
*******************************************************************************/
public BackendVariantsConfig withVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction)
{
this.variantRecordLookupFunction = variantRecordLookupFunction;
return (this);
}
}

106
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java Normal file
View File

@ -0,0 +1,106 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
import java.io.Serializable;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
/*******************************************************************************
** Utility methods for backends working with Variants.
*******************************************************************************/
public class BackendVariantsUtil
{
/*******************************************************************************
** Get the variant id from the session for the backend.
*******************************************************************************/
public static Serializable getVariantId(QBackendMetaData backendMetaData) throws QException
{
QSession session = QContext.getQSession();
String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey))
{
throw (new QException("Could not find Backend Variant information in session under key '" + variantTypeKey + "' for Backend '" + backendMetaData.getName() + "'"));
}
Serializable variantId = session.getBackendVariants().get(variantTypeKey);
return variantId;
}
/*******************************************************************************
** For backends that use variants, look up the variant record (in theory, based
** on an id in the session's backend variants map, then fetched from the backend's
** variant options table.
*******************************************************************************/
@SuppressWarnings("unchecked")
public static QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException
{
Serializable variantId = getVariantId(backendMetaData);
QRecord record;
if(backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction() != null)
{
Object o = QCodeLoader.getAdHoc(Object.class, backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction());
if(o instanceof UnsafeFunction<?,?,?> unsafeFunction)
{
record = ((UnsafeFunction<Serializable, QRecord, QException>) unsafeFunction).apply(variantId);
}
else if(o instanceof Function<?,?> function)
{
record = ((Function<Serializable, QRecord>) function).apply(variantId);
}
else
{
throw (new QException("Backend Variant's recordLookupFunction is not of any expected type (should have been caught by instance validation??)"));
}
}
else
{
GetInput getInput = new GetInput();
getInput.setShouldMaskPasswords(false);
getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
getInput.setPrimaryKey(variantId);
GetOutput getOutput = new GetAction().execute(getInput);
record = getOutput.getRecord();
}
if(record == null)
{
throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'"));
}
return record;
}
}

39
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java Normal file
View File

@ -0,0 +1,39 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
/*******************************************************************************
** temporary class, while we migrate from original way that variants were set up
** e.g., by calling 'variantOptionsTableUsernameField', to the new way, using
** the BackendVariantsConfig which uses a map of enum constants.
**
** so when those deprecated setters are removed, this enum can be too.
*****************************************************************************/
public enum LegacyBackendVariantSetting implements BackendVariantSetting
{
USERNAME,
PASSWORD,
API_KEY,
CLIENT_ID,
CLIENT_SECRET
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java
View File

@ -55,6 +55,8 @@ public class BulkInsertExtractStep extends AbstractExtractStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
int rowsAdded = 0;
int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);

82
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java
View File

@ -22,16 +22,38 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertLoadStep extends LoadViaInsertStep
public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSummaryProviderInterface
{
private static final QLogger LOG = QLogger.getLogger(BulkInsertLoadStep.class);
private Serializable firstInsertedPrimaryKey = null;
private Serializable lastInsertedPrimaryKey = null;
/*******************************************************************************
**
@ -42,4 +64,62 @@ public class BulkInsertLoadStep extends LoadViaInsertStep
return (QInputSource.USER);
}
/***************************************************************************
**
***************************************************************************/
@Override
public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
super.runOnePage(runBackendStepInput, runBackendStepOutput);
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName"));
List<QRecord> insertedRecords = runBackendStepOutput.getRecords();
for(QRecord insertedRecord : insertedRecords)
{
if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
{
if(firstInsertedPrimaryKey == null)
{
firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
}
lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
}
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
{
ArrayList<ProcessSummaryLineInterface> processSummary = getTransformStep().getProcessSummary(runBackendStepOutput, isForResultScreen);
try
{
if(firstInsertedPrimaryKey != null)
{
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepOutput.getValueString("tableName"));
QFieldMetaData field = table.getField(table.getPrimaryKeyField());
if(field.getType().isNumeric())
{
ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey);
idsLine.setCount(null);
processSummary.add(idsLine);
}
}
}
catch(Exception e)
{
LOG.warn("Error adding inserted-keys process summary line", e);
}
return (processSummary);
}
}

72
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java
View File

@ -32,16 +32,20 @@ import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
@ -63,12 +67,37 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
{
try
{
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
QRecord savedBulkLoadProfileRecord = BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
BulkLoadProfile bulkLoadProfile;
if(BulkInsertStepUtils.isHeadless(runBackendStepInput))
{
//////////////////////////////////////////////////////////////////////////////
// if running headless, build bulkLoadProfile from the saved profile record //
//////////////////////////////////////////////////////////////////////////////
if(savedBulkLoadProfileRecord == null)
{
throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load"));
}
SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
try
{
bulkLoadProfile = JsonUtils.toObject(savedBulkLoadProfile.getMappingJson(), BulkLoadProfile.class);
}
catch(Exception e)
{
throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e));
}
}
else
{
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
}
/////////////////////////////////////////////////////////////////////////
// put the list of bulk load profile into the process state - it's the //
@ -183,21 +212,32 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
/////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping))
if(BulkInsertStepUtils.isHeadless(runBackendStepInput))
{
//////////////////////////////////////////////////////////////////////////////////
// just go to the prepareValueMapping backend step - it'll figure out the rest. //
// it's also where the value-mapping loop of steps points. //
// and, this will actually be the default (e.g., the step after this one). //
//////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue("valueMappingFieldIndex", -1);
////////////////////////////////////////////////////////////////////////
// if running headless, always go straight to the preview screen next //
// todo actually, we could make this execute, right? //
////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
}
else
{
//////////////////////////////////////////////////////////////////////////////////
// else - if no values to map - continue with the standard streamed-ETL preview //
//////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping))
{
//////////////////////////////////////////////////////////////////////////////////
// just go to the prepareValueMapping backend step - it'll figure out the rest. //
// it's also where the value-mapping loop of steps points. //
// and, this will actually be the default (e.g., the step after this one). //
//////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue("valueMappingFieldIndex", -1);
}
else
{
//////////////////////////////////////////////////////////////////////////////////
// else - if no values to map - continue with the standard streamed-ETL preview //
//////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
}
}
}
catch(Exception e)

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java
View File

@ -29,11 +29,13 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerKeyRecordMessage;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.json.JSONArray;
@ -69,6 +71,18 @@ public class BulkInsertStepUtils
/***************************************************************************
**
***************************************************************************/
public static void setStorageInputForTheFile(RunProcessInput runProcessInput, StorageInput storageInput)
{
ArrayList<StorageInput> storageInputs = new ArrayList<>();
storageInputs.add(storageInput);
runProcessInput.addValue("theFile", storageInputs);
}
/***************************************************************************
**
***************************************************************************/
@ -144,13 +158,62 @@ public class BulkInsertStepUtils
/***************************************************************************
**
***************************************************************************/
public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
public static QRecord handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId");
if(savedBulkLoadProfileId != null)
{
QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId);
runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord);
return (savedBulkLoadProfileRecord);
}
return (null);
}
/***************************************************************************
**
***************************************************************************/
public static boolean isHeadless(RunBackendStepInput runBackendStepInput)
{
return (runBackendStepInput.getValuePrimitiveBoolean("isHeadless"));
}
/***************************************************************************
**
***************************************************************************/
public static void setHeadless(RunProcessInput runProcessInput)
{
runProcessInput.addValue("isHeadless", true);
}
/***************************************************************************
**
***************************************************************************/
public static void setProcessTracerKeyRecordMessage(RunProcessInput runProcessInput, ProcessTracerKeyRecordMessage processTracerKeyRecordMessage)
{
runProcessInput.addValue("processTracerKeyRecordMessage", processTracerKeyRecordMessage);
}
/***************************************************************************
**
***************************************************************************/
public static ProcessTracerKeyRecordMessage getProcessTracerKeyRecordMessage(RunBackendStepInput runBackendStepInput)
{
Serializable value = runBackendStepInput.getValue("processTracerKeyRecordMessage");
if(value instanceof ProcessTracerKeyRecordMessage processTracerKeyRecordMessage)
{
return (processTracerKeyRecordMessage);
}
return (null);
}
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java
View File

@ -258,6 +258,11 @@ public class BulkLoadValueMapper
valuesNotFound.add(value);
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// todo - we should probably be doing a lot of what QJavalinImplementation.finishPossibleValuesRequest does here //
// to apply possible-value filters. difficult to pass values in, but needed... //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
searchPossibleValueSourceInput.setIdList(idList);
searchPossibleValueSourceInput.setLimit(values.size());
LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0)));

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
View File

@ -239,6 +239,10 @@ public class ColumnStatsStep implements BackendStep
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, queryJoin == null ? null : List.of(queryJoin), null);
/////////////////////////////////////////////////////////////////////////////////////////////////
// todo - be aware of possible name collisions here!! (e.g., a table w/ a field named `count`) //
/////////////////////////////////////////////////////////////////////////////////////////////////
QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "count", countField), valueCounts);
runBackendStepOutput.addValue("valueCounts", valueCounts);

66
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java Normal file
View File

@ -0,0 +1,66 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.tracing;
/*******************************************************************************
** Specialization of process tracer message, to indicate a 'key record' that was
** used as an input or trigger to a process.
*******************************************************************************/
public class ProcessTracerKeyRecordMessage extends ProcessTracerMessage
{
private final String tableName;
private final Integer recordId;
/***************************************************************************
**
***************************************************************************/
public ProcessTracerKeyRecordMessage(String tableName, Integer recordId)
{
super("Process Key Record is " + tableName + " " + recordId);
this.tableName = tableName;
this.recordId = recordId;
}
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getTableName()
{
return (this.tableName);
}
/*******************************************************************************
** Getter for recordId
*******************************************************************************/
public Integer getRecordId()
{
return (this.recordId);
}
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java
View File

@ -22,13 +22,16 @@
package com.kingsrook.qqq.backend.core.processes.tracing;
import java.io.Serializable;
/*******************************************************************************
** Basic class that can be passed in to ProcessTracerInterface.handleMessage.
** This class just provides for a string message. We anticipate subclasses
** that may have more specific data, that specific tracer implementations may
** be aware of.
*******************************************************************************/
public class ProcessTracerMessage
public class ProcessTracerMessage implements Serializable
{
private String message;

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java
View File

@ -441,11 +441,16 @@ public class QScheduleManager
try
{
HashMap<String, Serializable> parameters = new HashMap<>(paramMap);
HashMap<String, Serializable> variantMap = new HashMap<>(Map.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())));
String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
String variantOptionsTableName = backendMetaData.getBackendVariantsConfig().getOptionsTableName();
String variantOptionsTableIdFieldName = QContext.getQInstance().getTable(variantOptionsTableName).getPrimaryKeyField();
HashMap<String, Serializable> variantMap = new HashMap<>(Map.of(variantTypeKey, qRecord.getValue(variantOptionsTableIdFieldName)));
parameters.put("backendVariantData", variantMap);
String identity = schedulableIdentity.getIdentity() + ";" + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField());
String description = schedulableIdentity.getDescription() + " for variant: " + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField());
String identity = schedulableIdentity.getIdentity() + ";" + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName);
String description = schedulableIdentity.getDescription() + " for variant: " + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName);
BasicSchedulableIdentity variantIdentity = new BasicSchedulableIdentity(identity, description);

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java
View File

@ -34,9 +34,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -102,7 +99,8 @@ public class SchedulerUtils
try
{
QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend());
Map<String, Serializable> thisVariantData = MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()));
QTableMetaData variantTable = QContext.getQInstance().getTable(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
Map<String, Serializable> thisVariantData = MapBuilder.of(backendMetaData.getBackendVariantsConfig().getVariantTypeKey(), qRecord.getValue(variantTable.getPrimaryKeyField()));
executeSingleProcess(process, thisVariantData, processInputValues);
}
catch(Exception e)
@ -181,8 +179,8 @@ public class SchedulerUtils
QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(processMetaData.getVariantBackend());
QueryInput queryInput = new QueryInput();
queryInput.setTableName(backendMetaData.getVariantOptionsTableName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue())));
queryInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
queryInput.setFilter(backendMetaData.getBackendVariantsConfig().getOptionsFilter());
QueryOutput queryOutput = new QueryAction().execute(queryInput);
records = queryOutput.getRecords();

144
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
View File

@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Consumer;
@ -55,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
@ -93,12 +95,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@ -182,6 +187,143 @@ public class QInstanceValidatorTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBackendVariants()
{
BackendVariantSetting setting = new BackendVariantSetting() {};
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)),
"Missing backendVariantsConfig in backend [variant] which is marked as usesVariants");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(false)
.withBackendVariantsConfig(new BackendVariantsConfig())),
"Should not have a backendVariantsConfig");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(null)
.withBackendVariantsConfig(new BackendVariantsConfig())),
"Should not have a backendVariantsConfig");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig())),
"Missing variantTypeKey in backendVariantsConfig",
"Missing optionsTableName in backendVariantsConfig",
"Missing or empty backendSettingSourceFieldNameMap");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName("notATable")
.withBackendSettingSourceFieldNameMap(Map.of(setting, "field")))),
"Unrecognized optionsTableName [notATable] in backendVariantsConfig");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withOptionsFilter(new QQueryFilter(new QFilterCriteria("notAField", QCriteriaOperator.EQUALS, 1)))
.withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")))),
"optionsFilter in backendVariantsConfig in backend [variant]: Criteria fieldName notAField is not a field");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withBackendSettingSourceFieldNameMap(Map.of(setting, "noSuchField")))),
"Unrecognized fieldName [noSuchField] in backendSettingSourceFieldNameMap");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withVariantRecordLookupFunction(new QCodeReference(CustomizerThatIsNotOfTheRightBaseClass.class))
.withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier"))
)),
"VariantRecordSupplier in backendVariantsConfig in backend [variant]: CodeReference is not any of the expected types: com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction, java.util.function.Function");
assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName"))
)));
assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withVariantRecordLookupFunction(new QCodeReference(VariantRecordFunction.class))
.withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier"))
)));
assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withVariantRecordLookupFunction(new QCodeReference(VariantRecordUnsafeFunction.class))
.withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier"))
)));
}
/***************************************************************************
**
***************************************************************************/
public static class VariantRecordFunction implements Function<Serializable, QRecord>
{
/***************************************************************************
**
***************************************************************************/
@Override
public QRecord apply(Serializable serializable)
{
return null;
}
}
/***************************************************************************
**
***************************************************************************/
public static class VariantRecordUnsafeFunction implements UnsafeFunction<Serializable, QRecord, QException>
{
/***************************************************************************
**
***************************************************************************/
@Override
public QRecord apply(Serializable serializable) throws QException
{
return null;
}
}
/*******************************************************************************
** Test an instance with null tables - should throw.
**
@ -2369,7 +2511,7 @@ public class QInstanceValidatorTest extends BaseTest
{
int noOfReasons = actualReasons == null ? 0 : actualReasons.size();
assertEquals(expectedReasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", expectedReasons)
+ "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--"));
+ "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--"));
}
for(String reason : expectedReasons)

44
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java Normal file
View File

@ -0,0 +1,44 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for EmptyMetaDataProducerOutput
*******************************************************************************/
class EmptyMetaDataProducerOutputTest
{
/*******************************************************************************
** sorry, just here to avoid a dip in coverage.
*******************************************************************************/
@Test
void test()
{
QInstance qInstance = new QInstance();
new EmptyMetaDataProducerOutput().addSelfToInstance(qInstance);
}
}

62
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java Normal file
View File

@ -0,0 +1,62 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for SectionFactory
*******************************************************************************/
class SectionFactoryTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test()
{
QFieldSection t1section = SectionFactory.defaultT1("id", "name");
assertEquals(SectionFactory.getDefaultT1name(), t1section.getName());
assertEquals(SectionFactory.getDefaultT1iconName(), t1section.getIcon().getName());
assertEquals(Tier.T1, t1section.getTier());
assertEquals(List.of("id", "name"), t1section.getFieldNames());
QFieldSection t2section = SectionFactory.defaultT2("size", "age");
assertEquals(SectionFactory.getDefaultT2name(), t2section.getName());
assertEquals(SectionFactory.getDefaultT2iconName(), t2section.getIcon().getName());
assertEquals(Tier.T2, t2section.getTier());
assertEquals(List.of("size", "age"), t2section.getFieldNames());
QFieldSection t3section = SectionFactory.defaultT3("createDate", "modifyDate");
assertEquals(SectionFactory.getDefaultT3name(), t3section.getName());
assertEquals(SectionFactory.getDefaultT3iconName(), t3section.getIcon().getName());
assertEquals(Tier.T3, t3section.getTier());
assertEquals(List.of("createDate", "modifyDate"), t3section.getFieldNames());
}
}

100
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java Normal file
View File

@ -0,0 +1,100 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for BackendVariantsUtil
*******************************************************************************/
class BackendVariantsUtilTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetVariantId() throws QException
{
QBackendMetaData myBackend = getBackendMetaData();
assertThatThrownBy(() -> BackendVariantsUtil.getVariantId(myBackend))
.hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'");
QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701));
assertEquals(1701, BackendVariantsUtil.getVariantId(myBackend));
}
/***************************************************************************
**
***************************************************************************/
private static QBackendMetaData getBackendMetaData()
{
QBackendMetaData myBackend = new QBackendMetaData()
.withName("TestBackend")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withOptionsTableName(TestUtils.TABLE_NAME_SHAPE)
.withVariantTypeKey("yourSelectedShape"));
return myBackend;
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetVariantRecord() throws QException
{
QBackendMetaData myBackend = getBackendMetaData();
TestUtils.insertDefaultShapes(QContext.getQInstance());
assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend))
.hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'");
QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701));
assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend))
.hasMessageContaining("Could not find Backend Variant in table shape with id '1701'");
QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1));
QRecord variantRecord = BackendVariantsUtil.getVariantRecord(myBackend);
assertEquals(1, variantRecord.getValueInteger("id"));
assertNotNull(variantRecord.getValue("name"));
}
}

1
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java
View File

@ -61,7 +61,6 @@ public abstract class AbstractAPIAction
apiActionUtil.setBackendMetaData(this.backendMetaData);
apiActionUtil.setActionInput(actionInput);
apiActionUtil.setSession(session);
}
}

80
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java
View File

@ -36,7 +36,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -61,6 +60,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -77,6 +79,7 @@ import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorExcep
import com.kingsrook.qqq.backend.module.api.model.AuthorizationType;
import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog;
import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData;
import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendVariantSetting;
import com.kingsrook.qqq.backend.module.api.model.metadata.APITableBackendDetails;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.BooleanUtils;
@ -114,7 +117,6 @@ public class BaseAPIActionUtil
{
private final QLogger LOG = QLogger.getLogger(BaseAPIActionUtil.class);
protected QSession session; // todo not commit - delete!!
protected APIBackendMetaData backendMetaData;
protected AbstractTableActionInput actionInput;
@ -777,8 +779,8 @@ public class BaseAPIActionUtil
{
if(backendMetaData.getUsesVariants())
{
QRecord record = getVariantRecord();
return (record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData);
return (record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.API_KEY, APIBackendVariantSetting.API_KEY)));
}
return (backendMetaData.getApiKey());
@ -786,6 +788,18 @@ public class BaseAPIActionUtil
/***************************************************************************
** todo - once deprecated variant methods are removed from QBackendMetaData,
** then we can remove the LegacyBackendVariantSetting enum, and this param.
***************************************************************************/
private String getVariantSettingSourceFieldName(APIBackendMetaData backendMetaData, LegacyBackendVariantSetting legacyBackendVariantSetting, APIBackendVariantSetting apiBackendVariantSetting)
{
Map<BackendVariantSetting, String> map = CollectionUtils.nonNullMap(backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap());
return map.getOrDefault(legacyBackendVariantSetting, map.get(apiBackendVariantSetting));
}
/***************************************************************************
**
***************************************************************************/
@ -793,8 +807,11 @@ public class BaseAPIActionUtil
{
if(backendMetaData.getUsesVariants())
{
QRecord record = getVariantRecord();
return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData);
return (Pair.of(
record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.USERNAME, APIBackendVariantSetting.USERNAME)),
record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.PASSWORD, APIBackendVariantSetting.PASSWORD))
));
}
return (Pair.of(backendMetaData.getUsername(), backendMetaData.getPassword()));
@ -802,46 +819,6 @@ public class BaseAPIActionUtil
/*******************************************************************************
** For backends that use variants, look up the variant record (in theory, based
** on an id in the session's backend variants map, then fetched from the backend's
** variant options table.
*******************************************************************************/
protected QRecord getVariantRecord() throws QException
{
Serializable variantId = getVariantId();
GetInput getInput = new GetInput();
getInput.setShouldMaskPasswords(false);
getInput.setTableName(backendMetaData.getVariantOptionsTableName());
getInput.setPrimaryKey(variantId);
GetOutput getOutput = new GetAction().execute(getInput);
QRecord record = getOutput.getRecord();
if(record == null)
{
throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'"));
}
return record;
}
/*******************************************************************************
** Get the variant id from the session for the backend.
*******************************************************************************/
protected Serializable getVariantId() throws QException
{
QSession session = QContext.getQSession();
if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
{
throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"));
}
Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
return variantId;
}
/*******************************************************************************
**
*******************************************************************************/
@ -854,7 +831,7 @@ public class BaseAPIActionUtil
String accessTokenKey = "accessToken";
if(backendMetaData.getUsesVariants())
{
Serializable variantId = getVariantId();
Serializable variantId = BackendVariantsUtil.getVariantId(backendMetaData);
accessTokenKey = accessTokenKey + ":" + variantId;
}
@ -944,8 +921,11 @@ public class BaseAPIActionUtil
{
if(backendMetaData.getUsesVariants())
{
QRecord record = getVariantRecord();
return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableClientIdField()), record.getValueString(backendMetaData.getVariantOptionsTableClientSecretField())));
QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData);
return (Pair.of(
record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_ID, APIBackendVariantSetting.CLIENT_ID)),
record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_SECRET, APIBackendVariantSetting.CLIENT_SECRET))
));
}
return (Pair.of(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
@ -1480,9 +1460,9 @@ public class BaseAPIActionUtil
** Setter for session
**
*******************************************************************************/
@Deprecated(since = "wasn't used.")
public void setSession(QSession session)
{
this.session = session;
}

38
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java Normal file
View File

@ -0,0 +1,38 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.api.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
/*******************************************************************************
** settings that the API backend module can get from a backend variant.
*******************************************************************************/
public enum APIBackendVariantSetting implements BackendVariantSetting
{
USERNAME,
PASSWORD,
API_KEY,
CLIENT_ID,
CLIENT_SECRET
}

24
qqq-backend-module-filesystem/pom.xml
View File

@ -50,6 +50,17 @@
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.261</version>
</dependency>
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-sftp</artifactId>
<version>2.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-sftp</artifactId>
<version>2.14.0</version>
</dependency>
<dependency>
<groupId>cloud.localstack</groupId>
<artifactId>localstack-utils</artifactId>
@ -57,6 +68,19 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>1.15.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>5.7.0</version>
<scope>test</scope>
</dependency>
<!-- Common deps for all qqq modules -->
<dependency>
<groupId>org.apache.maven.plugins</groupId>

426
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -25,9 +25,13 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
@ -38,6 +42,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
@ -47,12 +53,19 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -68,6 +81,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
{
private static final QLogger LOG = QLogger.getLogger(AbstractBaseFilesystemAction.class);
protected QRecord backendVariantRecord = null;
/*******************************************************************************
@ -80,6 +95,21 @@ public abstract class AbstractBaseFilesystemAction<FILE>
/***************************************************************************
** get the size of the specified file, null if not supported/available
***************************************************************************/
public abstract Long getFileSize(FILE file);
/***************************************************************************
** get the createDate of the specified file, null if not supported/available
***************************************************************************/
public abstract Instant getFileCreateDate(FILE file);
/***************************************************************************
** get the createDate of the specified file, null if not supported/available
***************************************************************************/
public abstract Instant getFileModifyDate(FILE file);
/*******************************************************************************
** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
*******************************************************************************/
@ -116,13 +146,21 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/
public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException;
/*******************************************************************************
** e.g., with a base path of /foo/
** and a table path of /bar/
** and a file at /foo/bar/baz.txt
** give us just the baz.txt part.
*******************************************************************************/
public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable);
public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table)
{
String tablePath = getFullBasePath(table, backend);
String strippedPath = filePath.replaceFirst(".*" + tablePath, "");
String withoutLeadingSlash = stripLeadingSlash(strippedPath); // todo - dangerous, do all backends really want this??
return (withoutLeadingSlash);
}
@ -133,7 +171,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase)
{
AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase);
String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : "";
String basePath = metaData.getBasePath();
if(backendBase.getUsesVariants())
{
Map<BackendVariantSetting, String> fieldNameMap = backendBase.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap();
if(fieldNameMap.containsKey(SFTPBackendVariantSetting.BASE_PATH))
{
basePath = backendVariantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.BASE_PATH));
}
}
String fullPath = StringUtils.hasContent(basePath) ? basePath : "";
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
if(StringUtils.hasContent(tableDetails.getBasePath()))
@ -164,6 +212,34 @@ public abstract class AbstractBaseFilesystemAction<FILE>
/*******************************************************************************
**
*******************************************************************************/
public static String stripLeadingSlash(String path)
{
if(path == null)
{
return (null);
}
return (path.replaceFirst("^/+", ""));
}
/*******************************************************************************
**
*******************************************************************************/
public static String stripTrailingSlash(String path)
{
if(path == null)
{
return (null);
}
return (path.replaceFirst("/+$", ""));
}
/*******************************************************************************
** Get the backend metaData, type-checked as the requested type.
*******************************************************************************/
@ -208,106 +284,219 @@ public abstract class AbstractBaseFilesystemAction<FILE>
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
List<FILE> files = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
int recordCount = 0;
FILE_LOOP:
for(FILE file : files)
switch(tableDetails.getCardinality())
{
InputStream inputStream = readFile(file);
switch(tableDetails.getCardinality())
{
case MANY:
{
LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file)));
switch(tableDetails.getRecordFormat())
{
case CSV:
{
String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
fileContents = customizeFileContentsAfterReading(table, fileContents);
if(queryInput.getRecordPipe() != null)
{
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
{
////////////////////////////////////////////////////////////////////////////////////////////
// Before the records go into the pipe, make sure their backend details are added to them //
////////////////////////////////////////////////////////////////////////////////////////////
addBackendDetailsToRecord(record, file);
}));
}
else
{
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
}
break;
}
case JSON:
{
String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
fileContents = customizeFileContentsAfterReading(table, fileContents);
// todo - pipe support!!
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
break;
}
default:
{
throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
}
}
break;
}
case ONE:
{
////////////////////////////////////////////////////////////////////////////////
// for one-record tables, put the entire file's contents into a single record //
////////////////////////////////////////////////////////////////////////////////
String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
byte[] bytes = inputStream.readAllBytes();
QRecord record = new QRecord()
.withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
.withValue(tableDetails.getContentsFieldName(), bytes);
queryOutput.addRecord(record);
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
recordCount++;
////////////////////////////////////////////////////////////////////////////
// break out of the file loop if we have hit the limit (if one was given) //
////////////////////////////////////////////////////////////////////////////
if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null)
{
if(recordCount >= queryInput.getFilter().getLimit())
{
break FILE_LOOP;
}
}
break;
}
default:
{
throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
}
}
case MANY -> completeExecuteQueryForManyTable(queryInput, queryOutput, files, table, tableDetails);
case ONE -> completeExecuteQueryForOneTable(queryInput, queryOutput, files, table, tableDetails);
default -> throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
}
return queryOutput;
return (queryOutput);
}
catch(Exception e)
{
LOG.warn("Error executing query", e);
throw new QException("Error executing query", e);
}
finally
{
postAction();
}
}
/***************************************************************************
**
***************************************************************************/
private void setRecordValueIfFieldNameHasContent(QRecord record, String fieldName, UnsafeSupplier<Serializable, ?> valueSupplier)
{
if(StringUtils.hasContent(fieldName))
{
try
{
record.setValue(fieldName, valueSupplier.get());
}
catch(Exception e)
{
LOG.warn("Error setting record value for field", e, logPair("fieldName", fieldName));
}
}
}
/***************************************************************************
**
***************************************************************************/
private void completeExecuteQueryForOneTable(QueryInput queryInput, QueryOutput queryOutput, List<FILE> files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException
{
int recordCount = 0;
List<QRecord> records = new ArrayList<>();
for(FILE file : files)
{
////////////////////////////////////////////////////////////////////////////////
// for one-record tables, put the entire file's contents into a single record //
////////////////////////////////////////////////////////////////////////////////
String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
QRecord record = new QRecord();
setRecordValueIfFieldNameHasContent(record, tableDetails.getFileNameFieldName(), () -> filePathWithoutBase);
setRecordValueIfFieldNameHasContent(record, tableDetails.getBaseNameFieldName(), () -> stripAllPaths(filePathWithoutBase));
setRecordValueIfFieldNameHasContent(record, tableDetails.getSizeFieldName(), () -> getFileSize(file));
setRecordValueIfFieldNameHasContent(record, tableDetails.getCreateDateFieldName(), () -> getFileCreateDate(file));
setRecordValueIfFieldNameHasContent(record, tableDetails.getModifyDateFieldName(), () -> getFileModifyDate(file));
if(shouldHeavyFileContentsBeRead(queryInput, table, tableDetails))
{
try(InputStream inputStream = readFile(file))
{
byte[] bytes = inputStream.readAllBytes();
record.withValue(tableDetails.getContentsFieldName(), bytes);
}
catch(Exception e)
{
record.addError(new SystemErrorStatusMessage("Error reading file contents: " + e.getMessage()));
}
}
else
{
Long size = record.getValueLong(tableDetails.getSizeFieldName());
if(size != null)
{
if(record.getBackendDetails() == null)
{
record.setBackendDetails(new HashMap<>());
}
if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null)
{
record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>());
}
((Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size);
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// the listFiles method may have used a "path" criteria. //
// if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
QQueryFilter filterForRecords = queryInput.getFilter();
if(filterForRecords != null)
{
filterForRecords = filterForRecords.clone();
CollectionUtils.nonNullList(filterForRecords.getCriteria())
.removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria);
}
if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record))
{
records.add(record);
}
}
BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), records);
records = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), records);
queryOutput.addRecords(records);
}
/***************************************************************************
**
***************************************************************************/
private Serializable stripAllPaths(String filePath)
{
if(filePath == null)
{
return null;
}
return (filePath.replaceFirst(".*/", ""));
}
/***************************************************************************
**
***************************************************************************/
protected static boolean isPathEqualsCriteria(QFilterCriteria criteria)
{
return "path".equals(criteria.getFieldName()) && QCriteriaOperator.EQUALS.equals(criteria.getOperator());
}
/***************************************************************************
**
***************************************************************************/
private void completeExecuteQueryForManyTable(QueryInput queryInput, QueryOutput queryOutput, List<FILE> files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException, IOException
{
int recordCount = 0;
for(FILE file : files)
{
try(InputStream inputStream = readFile(file))
{
LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file)));
switch(tableDetails.getRecordFormat())
{
case CSV ->
{
String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
fileContents = customizeFileContentsAfterReading(table, fileContents);
if(queryInput.getRecordPipe() != null)
{
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
{
////////////////////////////////////////////////////////////////////////////////////////////
// Before the records go into the pipe, make sure their backend details are added to them //
////////////////////////////////////////////////////////////////////////////////////////////
addBackendDetailsToRecord(record, file);
}));
}
else
{
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
}
}
case JSON ->
{
String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
fileContents = customizeFileContentsAfterReading(table, fileContents);
// todo - pipe support!!
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
}
default -> throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
}
}
}
}
/***************************************************************************
**
***************************************************************************/
private static boolean shouldHeavyFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails)
{
boolean doReadContents = true;
if(table.getField(tableDetails.getContentsFieldName()).getIsHeavy())
{
if(!queryInput.getShouldFetchHeavyFields())
{
doReadContents = false;
}
}
return doReadContents;
}
@ -319,7 +508,16 @@ public abstract class AbstractBaseFilesystemAction<FILE>
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(countInput.getTableName());
queryInput.setFilter(countInput.getFilter());
QQueryFilter filter = countInput.getFilter();
if(filter != null)
{
filter = filter.clone();
filter.setSkip(null);
filter.setLimit(null);
}
queryInput.setFilter(filter);
QueryOutput queryOutput = executeQuery(queryInput);
CountOutput countOutput = new CountOutput();
@ -353,14 +551,25 @@ public abstract class AbstractBaseFilesystemAction<FILE>
** Method that subclasses can override to add pre-action things (e.g., setting up
** s3 client).
*******************************************************************************/
public void preAction(QBackendMetaData backendMetaData)
public void preAction(QBackendMetaData backendMetaData) throws QException
{
/////////////////////////////////////////////////////////////////////
// noop in base class - subclasses can add functionality if needed //
/////////////////////////////////////////////////////////////////////
if(backendMetaData.getUsesVariants())
{
this.backendVariantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData);
}
}
/***************************************************************************
** Method that subclasses can override to add post-action things (e.g., closing resources)
***************************************************************************/
public void postAction()
{
//////////////////
// noop in base //
//////////////////
}
/*******************************************************************************
**
@ -411,10 +620,18 @@ public abstract class AbstractBaseFilesystemAction<FILE>
{
for(QRecord record : insertInput.getRecords())
{
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
output.addRecord(record);
try
{
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
output.addRecord(record);
}
catch(Exception e)
{
record.addError(new SystemErrorStatusMessage("Error writing file: " + e.getMessage()));
output.addRecord(record);
}
}
}
else
@ -428,5 +645,10 @@ public abstract class AbstractBaseFilesystemAction<FILE>
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
finally
{
postAction();
}
}
}

128
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java
View File

@ -41,6 +41,10 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
private String contentsFieldName;
private String fileNameFieldName;
private String baseNameFieldName;
private String sizeFieldName;
private String createDateFieldName;
private String modifyDateFieldName;
@ -281,4 +285,128 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
}
}
/*******************************************************************************
** Getter for sizeFieldName
*******************************************************************************/
public String getSizeFieldName()
{
return (this.sizeFieldName);
}
/*******************************************************************************
** Setter for sizeFieldName
*******************************************************************************/
public void setSizeFieldName(String sizeFieldName)
{
this.sizeFieldName = sizeFieldName;
}
/*******************************************************************************
** Fluent setter for sizeFieldName
*******************************************************************************/
public AbstractFilesystemTableBackendDetails withSizeFieldName(String sizeFieldName)
{
this.sizeFieldName = sizeFieldName;
return (this);
}
/*******************************************************************************
** Getter for createDateFieldName
*******************************************************************************/
public String getCreateDateFieldName()
{
return (this.createDateFieldName);
}
/*******************************************************************************
** Setter for createDateFieldName
*******************************************************************************/
public void setCreateDateFieldName(String createDateFieldName)
{
this.createDateFieldName = createDateFieldName;
}
/*******************************************************************************
** Fluent setter for createDateFieldName
*******************************************************************************/
public AbstractFilesystemTableBackendDetails withCreateDateFieldName(String createDateFieldName)
{
this.createDateFieldName = createDateFieldName;
return (this);
}
/*******************************************************************************
** Getter for modifyDateFieldName
*******************************************************************************/
public String getModifyDateFieldName()
{
return (this.modifyDateFieldName);
}
/*******************************************************************************
** Setter for modifyDateFieldName
*******************************************************************************/
public void setModifyDateFieldName(String modifyDateFieldName)
{
this.modifyDateFieldName = modifyDateFieldName;
}
/*******************************************************************************
** Fluent setter for modifyDateFieldName
*******************************************************************************/
public AbstractFilesystemTableBackendDetails withModifyDateFieldName(String modifyDateFieldName)
{
this.modifyDateFieldName = modifyDateFieldName;
return (this);
}
/*******************************************************************************
** Getter for baseNameFieldName
*******************************************************************************/
public String getBaseNameFieldName()
{
return (this.baseNameFieldName);
}
/*******************************************************************************
** Setter for baseNameFieldName
*******************************************************************************/
public void setBaseNameFieldName(String baseNameFieldName)
{
this.baseNameFieldName = baseNameFieldName;
}
/*******************************************************************************
** Fluent setter for baseNameFieldName
*******************************************************************************/
public AbstractFilesystemTableBackendDetails withBaseNameFieldName(String baseNameFieldName)
{
this.baseNameFieldName = baseNameFieldName;
return (this);
}
}

30
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java
View File

@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory;
import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails;
/*******************************************************************************
@ -64,6 +70,7 @@ public class FilesystemTableMetaDataBuilder
{
case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails();
case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails();
case SFTPBackendModule.BACKEND_TYPE -> new SFTPTableBackendDetails();
default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType());
};
@ -72,12 +79,31 @@ public class FilesystemTableMetaDataBuilder
.withIsHidden(true)
.withBackendName(backend.getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.INTEGER))
.withField(new QFieldMetaData("contents", QFieldType.STRING))
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("baseName", QFieldType.STRING))
.withField(new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))
.withField(new QFieldMetaData("contents", QFieldType.BLOB)
.withIsHeavy(true)
.withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD)
.withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s")
.withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName")
))
.withSection(SectionFactory.defaultT1("fileName"))
.withSection(SectionFactory.defaultT2("baseName", "contents", "size"))
.withSection(SectionFactory.defaultT3("createDate", "modifyDate"))
.withBackendDetails(tableBackendDetails
.withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")
.withBaseNameFieldName("baseName")
.withContentsFieldName("contents")
.withSizeFieldName("size")
.withCreateDateFieldName("createDate")
.withModifyDateFieldName("modifyDate")
.withBasePath(basePath)
.withGlob(glob));
}

46
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
View File

@ -35,6 +35,8 @@ import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -61,6 +63,50 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
/***************************************************************************
*
***************************************************************************/
@Override
public Long getFileSize(File file)
{
return (file.length());
}
/***************************************************************************
**
***************************************************************************/
@Override
public Instant getFileCreateDate(File file)
{
try
{
Path path = file.toPath();
BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class);
FileTime creationTime = attrs.creationTime();
return creationTime.toInstant();
}
catch(IOException e)
{
LOG.warn("Error getting file createDate", e, logPair("file", file));
return (null);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public Instant getFileModifyDate(File file)
{
return Instant.ofEpochMilli(file.lastModified());
}
/*******************************************************************************
** List the files for this table.
*******************************************************************************/

3
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java
View File

@ -63,7 +63,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -314,7 +313,7 @@ public class FilesystemImporterStep implements BackendStep
/*******************************************************************************
**
*******************************************************************************/
private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException
private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws QException
{
if(removeFileAfterImport)
{

13
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
@ -35,6 +36,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3CountAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
@ -112,6 +114,17 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke
/***************************************************************************
**
***************************************************************************/
@Override
public CountInterface getCountInterface()
{
return new S3CountAction();
}
/*******************************************************************************
**
*******************************************************************************/

50
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
import java.util.List;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
@ -56,11 +57,44 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
/***************************************************************************
**
***************************************************************************/
@Override
public Long getFileSize(S3ObjectSummary s3ObjectSummary)
{
return (s3ObjectSummary.getSize());
}
/***************************************************************************
**
***************************************************************************/
@Override
public Instant getFileCreateDate(S3ObjectSummary s3ObjectSummary)
{
return null;
}
/***************************************************************************
**
***************************************************************************/
@Override
public Instant getFileModifyDate(S3ObjectSummary s3ObjectSummary)
{
return s3ObjectSummary.getLastModified().toInstant();
}
/*******************************************************************************
** Setup the s3 utils object to be used for this action.
*******************************************************************************/
@Override
public void preAction(QBackendMetaData backendMetaData)
public void preAction(QBackendMetaData backendMetaData) throws QException
{
super.preAction(backendMetaData);
@ -179,20 +213,6 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
/*******************************************************************************
**
*******************************************************************************/
private String stripLeadingSlash(String path)
{
if(path == null)
{
return (null);
}
return (path.replaceFirst("^/+", ""));
}
/*******************************************************************************
** Get a string that represents the full path to a file.
*******************************************************************************/

170
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java Normal file
View File

@ -0,0 +1,170 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.AbstractSFTPAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPCountAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPDeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPInsertAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPQueryAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPStorageAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPUpdateAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails;
/*******************************************************************************
** QQQ Backend module for working with SFTP filesystems (as a client)
*******************************************************************************/
public class SFTPBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
{
public static final String BACKEND_TYPE = "sftp";
static
{
QBackendModuleDispatcher.registerBackendModule(new SFTPBackendModule());
}
/*******************************************************************************
** For filesystem backends, get the module-specific action base-class, that helps
** with functions like listing and deleting files.
*******************************************************************************/
@Override
public AbstractBaseFilesystemAction<SFTPDirEntryWithPath> getActionBase()
{
return (new AbstractSFTPAction());
}
/*******************************************************************************
** Method where a backend module must be able to provide its type (name).
*******************************************************************************/
@Override
public String getBackendType()
{
return (BACKEND_TYPE);
}
/*******************************************************************************
** Method to identify the class used for backend meta data for this module.
*******************************************************************************/
@Override
public Class<? extends QBackendMetaData> getBackendMetaDataClass()
{
return (SFTPBackendMetaData.class);
}
/*******************************************************************************
** Method to identify the class used for table-backend details for this module.
*******************************************************************************/
@Override
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
{
return (SFTPTableBackendDetails.class);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QueryInterface getQueryInterface()
{
return new SFTPQueryAction();
}
/***************************************************************************
**
***************************************************************************/
@Override
public CountInterface getCountInterface()
{
return new SFTPCountAction();
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public InsertInterface getInsertInterface()
{
return (new SFTPInsertAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public UpdateInterface getUpdateInterface()
{
return (new SFTPUpdateAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public DeleteInterface getDeleteInterface()
{
return (new SFTPDeleteAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QStorageInterface getStorageInterface()
{
return new SFTPStorageAction();
}
}

384
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java Normal file
View File

@ -0,0 +1,384 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting;
import org.apache.sshd.client.SshClient;
import org.apache.sshd.client.session.ClientSession;
import org.apache.sshd.sftp.client.SftpClient;
import org.apache.sshd.sftp.client.SftpClientFactory;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Base class for all SFTP filesystem actions
*******************************************************************************/
public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntryWithPath>
{
private static final QLogger LOG = QLogger.getLogger(AbstractSFTPAction.class);
/***************************************************************************
** singleton implementing Initialization-on-Demand Holder idiom
** to help ensure only a single SshClient object exists in a server.
***************************************************************************/
private static class SshClientManager
{
/***************************************************************************
**
***************************************************************************/
private static class Holder
{
private static final SshClient INSTANCE = SshClient.setUpDefaultClient();
static
{
INSTANCE.start();
}
}
/***************************************************************************
**
***************************************************************************/
public static SshClient getInstance()
{
return Holder.INSTANCE;
}
}
////////////////////////////////////////////////////////////////
// open clientSessionFirst, then sftpClient //
// and close them in reverse (sftpClient, then clientSession) //
////////////////////////////////////////////////////////////////
private ClientSession clientSession;
private SftpClient sftpClient;
/*******************************************************************************
** Set up the sftp utils object to be used for this action.
*******************************************************************************/
@Override
public void preAction(QBackendMetaData backendMetaData) throws QException
{
super.preAction(backendMetaData);
if(sftpClient != null)
{
LOG.debug("sftpClient object is already set - not re-setting it.");
return;
}
try
{
SFTPBackendMetaData sftpBackendMetaData = getBackendMetaData(SFTPBackendMetaData.class, backendMetaData);
String username = sftpBackendMetaData.getUsername();
String password = sftpBackendMetaData.getPassword();
String hostName = sftpBackendMetaData.getHostName();
Integer port = sftpBackendMetaData.getPort();
if(backendMetaData.getUsesVariants())
{
QRecord variantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData);
LOG.debug("Getting SFTP connection credentials from variant record",
logPair("tableName", backendMetaData.getBackendVariantsConfig().getOptionsTableName()),
logPair("id", variantRecord.getValue("id")),
logPair("name", variantRecord.getRecordLabel()));
Map<BackendVariantSetting, String> fieldNameMap = backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap();
if(fieldNameMap.containsKey(SFTPBackendVariantSetting.USERNAME))
{
username = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.USERNAME));
}
if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PASSWORD))
{
password = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.PASSWORD));
}
if(fieldNameMap.containsKey(SFTPBackendVariantSetting.HOSTNAME))
{
hostName = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.HOSTNAME));
}
if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PORT))
{
port = variantRecord.getValueInteger(fieldNameMap.get(SFTPBackendVariantSetting.PORT));
}
}
makeConnection(username, hostName, port, password);
}
catch(IOException e)
{
throw (new QException("Error setting up SFTP connection", e));
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public void postAction()
{
Consumer<AutoCloseable> closer = closable ->
{
if(closable != null)
{
try
{
closable.close();
}
catch(Exception e)
{
LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName()));
}
}
};
closer.accept(sftpClient);
closer.accept(clientSession);
}
/***************************************************************************
**
***************************************************************************/
protected SftpClient makeConnection(String username, String hostName, Integer port, String password) throws IOException
{
this.clientSession = SshClientManager.getInstance().connect(username, hostName, port).verify().getSession();
clientSession.addPasswordIdentity(password);
clientSession.auth().verify();
this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession);
return (this.sftpClient);
}
/***************************************************************************
**
***************************************************************************/
@Override
public Long getFileSize(SFTPDirEntryWithPath sftpDirEntryWithPath)
{
try
{
return sftpDirEntryWithPath.dirEntry().getAttributes().getSize();
}
catch(Exception e)
{
return (null);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public Instant getFileCreateDate(SFTPDirEntryWithPath sftpDirEntryWithPath)
{
try
{
return sftpDirEntryWithPath.dirEntry().getAttributes().getCreateTime().toInstant();
}
catch(Exception e)
{
return (null);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public Instant getFileModifyDate(SFTPDirEntryWithPath sftpDirEntryWithPath)
{
try
{
return sftpDirEntryWithPath.dirEntry().getAttributes().getModifyTime().toInstant();
}
catch(Exception e)
{
return (null);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public List<SFTPDirEntryWithPath> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
{
try
{
String fullPath = getFullBasePath(table, backendBase);
// todo - move somewhere shared
// todo - should all do this?
if(filter != null)
{
for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria()))
{
if(isPathEqualsCriteria(criteria))
{
fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar);
}
}
}
List<SFTPDirEntryWithPath> rs = new ArrayList<>();
for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath))
{
if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename()))
{
continue;
}
if(dirEntry.getAttributes().isDirectory())
{
// todo - recursive??
continue;
}
rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
}
return (rs);
}
catch(Exception e)
{
throw (new QException("Error listing files", e));
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public InputStream readFile(SFTPDirEntryWithPath dirEntry) throws IOException
{
return (sftpClient.read(getFullPathForFile(dirEntry)));
}
/***************************************************************************
**
***************************************************************************/
@Override
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
{
sftpClient.put(new ByteArrayInputStream(contents), path);
}
/***************************************************************************
**
***************************************************************************/
@Override
public String getFullPathForFile(SFTPDirEntryWithPath dirEntry)
{
return (dirEntry.path() + "/" + dirEntry.dirEntry().getFilename());
}
/***************************************************************************
**
***************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
throw (new QRuntimeException("Not yet implemented"));
}
/***************************************************************************
**
***************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
throw (new QRuntimeException("Not yet implemented"));
}
/***************************************************************************
*
***************************************************************************/
protected SftpClient getSftpClient(QBackendMetaData backend) throws QException
{
if(sftpClient == null)
{
preAction(backend);
}
return (sftpClient);
}
}

45
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPCountAction extends AbstractSFTPAction implements CountInterface
{
/*******************************************************************************
**
*******************************************************************************/
public CountOutput execute(CountInput countInput) throws QException
{
return (executeCount(countInput));
}
}

60
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java Normal file
View File

@ -0,0 +1,60 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPDeleteAction implements DeleteInterface
{
/*******************************************************************************
**
*******************************************************************************/
public DeleteOutput execute(DeleteInput deleteInput) throws QException
{
throw new NotImplementedException("SFTP delete not implemented");
/*
try
{
DeleteResult rs = new DeleteResult();
QTableMetaData table = deleteRequest.getTable();
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing delete: " + e.getMessage(), e);
}
*/
}
}

45
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPInsertAction extends AbstractSFTPAction implements InsertInterface
{
/*******************************************************************************
**
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
return (super.executeInsert(insertInput));
}
}

45
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPQueryAction extends AbstractSFTPAction implements QueryInterface
{
/*******************************************************************************
**
*******************************************************************************/
public QueryOutput execute(QueryInput queryInput) throws QException
{
return (super.executeQuery(queryInput));
}
}

157
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java Normal file
View File

@ -0,0 +1,157 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.utils.SFTPOutputStream;
import org.apache.sshd.sftp.client.SftpClient;
/*******************************************************************************
** (mass, streamed) storage action for sftp module
*******************************************************************************/
public class SFTPStorageAction extends AbstractSFTPAction implements QStorageInterface
{
/*******************************************************************************
** create an output stream in the storage backend - that can be written to,
** for the purpose of inserting or writing a file into storage.
*******************************************************************************/
@Override
public OutputStream createOutputStream(StorageInput storageInput) throws QException
{
try
{
SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend();
preAction(backend);
SftpClient sftpClient = getSftpClient(backend);
SFTPOutputStream sftpOutputStream = new SFTPOutputStream(sftpClient, getFullPath(storageInput));
return (sftpOutputStream);
}
catch(Exception e)
{
throw (new QException("Exception creating sftp output stream for file", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
private String getFullPath(StorageInput storageInput) throws QException
{
QTableMetaData table = storageInput.getTable();
QBackendMetaData backend = storageInput.getBackend();
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference());
/////////////////////////////////////////////////////////////
// s3 seems to do better w/o leading slashes, so, strip... //
/////////////////////////////////////////////////////////////
if(fullPath.startsWith("/"))
{
fullPath = fullPath.substring(1);
}
return fullPath;
}
/*******************************************************************************
** create an input stream in the storage backend - that can be read from,
** for the purpose of getting or reading a file from storage.
*******************************************************************************/
@Override
public InputStream getInputStream(StorageInput storageInput) throws QException
{
try
{
SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend();
preAction(backend);
SftpClient sftpClient = getSftpClient(backend);
InputStream inputStream = sftpClient.read(getFullPath(storageInput));
return (inputStream);
}
catch(Exception e)
{
throw (new QException("Exception getting sftp input stream for file.", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getDownloadURL(StorageInput storageInput) throws QException
{
try
{
throw new QRuntimeException("Not implemented");
//S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend();
//preAction(backend);
//
//AmazonS3 amazonS3 = getS3Utils().getAmazonS3();
//String fullPath = getFullPath(storageInput);
//return (amazonS3.getUrl(backend.getBucketName(), fullPath).toString());
}
catch(Exception e)
{
throw (new QException("Exception getting the sftp download URL.", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void makePublic(StorageInput storageInput) throws QException
{
try
{
throw new QRuntimeException("Not implemented");
}
catch(Exception e)
{
throw (new QException("Exception making sftp file publicly available.", e));
}
}
}

404
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java Normal file
View File

@ -0,0 +1,404 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.sshd.sftp.client.SftpClient;
/*******************************************************************************
** action for testing credentials for an SFTP backend connection
*******************************************************************************/
public class SFTPTestConnectionAction extends AbstractSFTPAction
{
/***************************************************************************
**
***************************************************************************/
public SFTPTestConnectionTestOutput testConnection(SFTPTestConnectionTestInput input)
{
try(SftpClient sftpClient = super.makeConnection(input.getUsername(), input.getHostName(), input.getPort(), input.getPassword()))
{
SFTPTestConnectionTestOutput output = new SFTPTestConnectionTestOutput().withIsConnectionSuccess(true);
if(StringUtils.hasContent(input.basePath))
{
try
{
Iterable<SftpClient.DirEntry> dirEntries = sftpClient.readDir(input.basePath);
/////////////////////////////////////////////////////////////////////////
// it seems like only the .iterator call throws if bad directory here. //
/////////////////////////////////////////////////////////////////////////
dirEntries.iterator();
output.setIsListBasePathSuccess(true);
}
catch(Exception e)
{
output.setIsListBasePathSuccess(false);
output.setListBasePathErrorMessage(e.getMessage());
}
}
return output;
}
catch(Exception e)
{
return new SFTPTestConnectionTestOutput().withIsConnectionSuccess(false).withConnectionErrorMessage(e.getMessage());
}
}
/***************************************************************************
**
***************************************************************************/
public static class SFTPTestConnectionTestInput
{
private String username;
private String hostName;
private Integer port;
private String password;
private String basePath;
/*******************************************************************************
** Getter for username
**
*******************************************************************************/
public String getUsername()
{
return username;
}
/*******************************************************************************
** Setter for username
**
*******************************************************************************/
public void setUsername(String username)
{
this.username = username;
}
/*******************************************************************************
** Fluent setter for username
**
*******************************************************************************/
public SFTPTestConnectionTestInput withUsername(String username)
{
this.username = username;
return (this);
}
/*******************************************************************************
** Getter for hostName
**
*******************************************************************************/
public String getHostName()
{
return hostName;
}
/*******************************************************************************
** Setter for hostName
**
*******************************************************************************/
public void setHostName(String hostName)
{
this.hostName = hostName;
}
/*******************************************************************************
** Fluent setter for hostName
**
*******************************************************************************/
public SFTPTestConnectionTestInput withHostName(String hostName)
{
this.hostName = hostName;
return (this);
}
/*******************************************************************************
** Getter for port
**
*******************************************************************************/
public Integer getPort()
{
return port;
}
/*******************************************************************************
** Setter for port
**
*******************************************************************************/
public void setPort(Integer port)
{
this.port = port;
}
/*******************************************************************************
** Fluent setter for port
**
*******************************************************************************/
public SFTPTestConnectionTestInput withPort(Integer port)
{
this.port = port;
return (this);
}
/*******************************************************************************
** Getter for password
**
*******************************************************************************/
public String getPassword()
{
return password;
}
/*******************************************************************************
** Setter for password
**
*******************************************************************************/
public void setPassword(String password)
{
this.password = password;
}
/*******************************************************************************
** Fluent setter for password
**
*******************************************************************************/
public SFTPTestConnectionTestInput withPassword(String password)
{
this.password = password;
return (this);
}
/*******************************************************************************
** Getter for basePath
**
*******************************************************************************/
public String getBasePath()
{
return basePath;
}
/*******************************************************************************
** Setter for basePath
**
*******************************************************************************/
public void setBasePath(String basePath)
{
this.basePath = basePath;
}
/*******************************************************************************
** Fluent setter for basePath
**
*******************************************************************************/
public SFTPTestConnectionTestInput withBasePath(String basePath)
{
this.basePath = basePath;
return (this);
}
}
/***************************************************************************
**
***************************************************************************/
public static class SFTPTestConnectionTestOutput
{
private Boolean isConnectionSuccess;
private String connectionErrorMessage;
private Boolean isListBasePathSuccess;
private String listBasePathErrorMessage;
/*******************************************************************************
** Getter for isSuccess
**
*******************************************************************************/
public Boolean getIsConnectionSuccess()
{
return isConnectionSuccess;
}
/*******************************************************************************
** Setter for isSuccess
**
*******************************************************************************/
public void setIsConnectionSuccess(Boolean isSuccess)
{
this.isConnectionSuccess = isSuccess;
}
/*******************************************************************************
** Fluent setter for isSuccess
**
*******************************************************************************/
public SFTPTestConnectionTestOutput withIsConnectionSuccess(Boolean isSuccess)
{
this.isConnectionSuccess = isSuccess;
return (this);
}
/*******************************************************************************
** Getter for connectionErrorMessage
**
*******************************************************************************/
public String getConnectionErrorMessage()
{
return connectionErrorMessage;
}
/*******************************************************************************
** Setter for connectionErrorMessage
**
*******************************************************************************/
public void setConnectionErrorMessage(String connectionErrorMessage)
{
this.connectionErrorMessage = connectionErrorMessage;
}
/*******************************************************************************
** Fluent setter for connectionErrorMessage
**
*******************************************************************************/
public SFTPTestConnectionTestOutput withConnectionErrorMessage(String connectionErrorMessage)
{
this.connectionErrorMessage = connectionErrorMessage;
return (this);
}
/*******************************************************************************
** Getter for listBasePathErrorMessage
*******************************************************************************/
public String getListBasePathErrorMessage()
{
return (this.listBasePathErrorMessage);
}
/*******************************************************************************
** Setter for listBasePathErrorMessage
*******************************************************************************/
public void setListBasePathErrorMessage(String listBasePathErrorMessage)
{
this.listBasePathErrorMessage = listBasePathErrorMessage;
}
/*******************************************************************************
** Fluent setter for listBasePathErrorMessage
*******************************************************************************/
public SFTPTestConnectionTestOutput withListBasePathErrorMessage(String listBasePathErrorMessage)
{
this.listBasePathErrorMessage = listBasePathErrorMessage;
return (this);
}
/*******************************************************************************
** Getter for isListBasePathSuccess
**
*******************************************************************************/
public Boolean getIsListBasePathSuccess()
{
return isListBasePathSuccess;
}
/*******************************************************************************
** Setter for isListBasePathSuccess
**
*******************************************************************************/
public void setIsListBasePathSuccess(Boolean isListBasePathSuccess)
{
this.isListBasePathSuccess = isListBasePathSuccess;
}
/*******************************************************************************
** Fluent setter for isListBasePathSuccess
**
*******************************************************************************/
public SFTPTestConnectionTestOutput withIsListBasePathSuccess(Boolean isListBasePathSuccess)
{
this.isListBasePathSuccess = isListBasePathSuccess;
return (this);
}
}
}

62
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java Normal file
View File

@ -0,0 +1,62 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPUpdateAction implements UpdateInterface
{
/*******************************************************************************
**
*******************************************************************************/
public UpdateOutput execute(UpdateInput updateInput) throws QException
{
throw new NotImplementedException("SFTP update not implemented");
/*
try
{
UpdateResult rs = new UpdateResult();
QTableMetaData table = updateRequest.getTable();
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing update: " + e.getMessage(), e);
}
*/
}
}

33
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java Normal file
View File

@ -0,0 +1,33 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.model;
import org.apache.sshd.sftp.client.SftpClient;
/*******************************************************************************
**
*******************************************************************************/
public record SFTPDirEntryWithPath(String path, SftpClient.DirEntry dirEntry)
{
}

198
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java Normal file
View File

@ -0,0 +1,198 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule;
/*******************************************************************************
** SFTP backend meta data.
*******************************************************************************/
public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData
{
private String username;
private String password;
private String hostName;
private Integer port;
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public SFTPBackendMetaData()
{
super();
setBackendType(SFTPBackendModule.class);
}
/*******************************************************************************
** Fluent setter for basePath
**
*******************************************************************************/
public SFTPBackendMetaData withBasePath(String basePath)
{
setBasePath(basePath);
return (this);
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public SFTPBackendMetaData withName(String name)
{
setName(name);
return this;
}
/*******************************************************************************
** Getter for username
*******************************************************************************/
public String getUsername()
{
return (this.username);
}
/*******************************************************************************
** Setter for username
*******************************************************************************/
public void setUsername(String username)
{
this.username = username;
}
/*******************************************************************************
** Fluent setter for username
*******************************************************************************/
public SFTPBackendMetaData withUsername(String username)
{
this.username = username;
return (this);
}
/*******************************************************************************
** Getter for password
*******************************************************************************/
public String getPassword()
{
return (this.password);
}
/*******************************************************************************
** Setter for password
*******************************************************************************/
public void setPassword(String password)
{
this.password = password;
}
/*******************************************************************************
** Fluent setter for password
*******************************************************************************/
public SFTPBackendMetaData withPassword(String password)
{
this.password = password;
return (this);
}
/*******************************************************************************
** Getter for hostName
*******************************************************************************/
public String getHostName()
{
return (this.hostName);
}
/*******************************************************************************
** Setter for hostName
*******************************************************************************/
public void setHostName(String hostName)
{
this.hostName = hostName;
}
/*******************************************************************************
** Fluent setter for hostName
*******************************************************************************/
public SFTPBackendMetaData withHostName(String hostName)
{
this.hostName = hostName;
return (this);
}
/*******************************************************************************
** Getter for port
*******************************************************************************/
public Integer getPort()
{
return (this.port);
}
/*******************************************************************************
** Setter for port
*******************************************************************************/
public void setPort(Integer port)
{
this.port = port;
}
/*******************************************************************************
** Fluent setter for port
*******************************************************************************/
public SFTPBackendMetaData withPort(Integer port)
{
this.port = port;
return (this);
}
}

38
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java Normal file
View File

@ -0,0 +1,38 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
/*******************************************************************************
**
*******************************************************************************/
public enum SFTPBackendVariantSetting implements BackendVariantSetting
{
USERNAME,
PASSWORD,
HOSTNAME,
PORT,
BASE_PATH
}

45
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule;
/*******************************************************************************
** SFTP specific Extension of QTableBackendDetails
*******************************************************************************/
public class SFTPTableBackendDetails extends AbstractFilesystemTableBackendDetails
{
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public SFTPTableBackendDetails()
{
super();
setBackendType(SFTPBackendModule.class);
}
}

179
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java Normal file
View File

@ -0,0 +1,179 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.utils;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.sshd.sftp.client.SftpClient;
import org.jetbrains.annotations.NotNull;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPOutputStream extends PipedOutputStream
{
private static final QLogger LOG = QLogger.getLogger(SFTPOutputStream.class);
private final SftpClient sftpClient;
private final PipedInputStream pipedInputStream;
private final Future<?> putFuture;
private AtomicBoolean started = new AtomicBoolean(false);
private AtomicReference<Exception> putException = new AtomicReference<>(null);
/***************************************************************************
**
***************************************************************************/
public SFTPOutputStream(SftpClient sftpClient, String path) throws IOException
{
pipedInputStream = new PipedInputStream(this, 32 * 1024);
this.sftpClient = sftpClient;
putFuture = Executors.newSingleThreadExecutor().submit(() ->
{
try
{
started.set(true);
sftpClient.put(pipedInputStream, path);
}
catch(Exception e)
{
putException.set(e);
LOG.error("Error putting SFTP output stream", e);
try
{
pipedInputStream.close();
}
catch(IOException ex)
{
LOG.error("Secondary error closing pipedInputStream after sftp put error", e);
}
throw new RuntimeException(e);
}
});
}
/***************************************************************************
**
***************************************************************************/
@Override
public void write(@NotNull byte[] b) throws IOException
{
try
{
super.write(b);
}
catch(IOException e)
{
if(putException.get() != null)
{
throw new IOException("Error performing SFTP put", putException.get());
}
throw new IOException("Error writing to SFTP output stream", e);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public void close() throws IOException
{
try
{
/////////////////////////////////////////////////////////////////////////////////////////////
// don't try to close anything until we know that the sftpClient.put call's thread //
// has tried to start (otherwise, race condition could cause us to close things too early) //
/////////////////////////////////////////////////////////////////////////////////////////////
int sleepLoops = 0;
while(!started.get() && sleepLoops++ <= 30)
{
SleepUtils.sleep(1, TimeUnit.SECONDS);
}
///////////////////////////////////////////////////////////////////////////////////////
// closing the pipedOutputStream (super) causes things to flush and complete the put //
///////////////////////////////////////////////////////////////////////////////////////
super.close();
////////////////////////////////
// wait for the put to finish //
////////////////////////////////
putFuture.get(60 - sleepLoops, TimeUnit.SECONDS);
///////////////////////////////////////////////////////////////////////////////
// in case the put-future never did start, throw explicitly mentioning that. //
///////////////////////////////////////////////////////////////////////////////
if(sleepLoops >= 30)
{
throw (new Exception("future to can sftpClient.put() was never started."));
}
}
catch(ExecutionException ee)
{
throw new IOException("Error performing SFTP put", ee);
}
catch(Exception e)
{
if(putException.get() != null)
{
throw new IOException("Error performing SFTP put", putException.get());
}
throw new IOException("Error closing SFTP output stream", e);
}
finally
{
try
{
sftpClient.close();
}
catch(IOException e)
{
LOG.error("Error closing SFTP client", e);
}
}
}
}

140
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.FilesystemTableMetaDataBuilder;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
@ -52,6 +55,10 @@ import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fil
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails;
import org.apache.commons.io.FileUtils;
@ -60,20 +67,26 @@ import org.apache.commons.io.FileUtils;
*******************************************************************************/
public class TestUtils
{
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String BACKEND_NAME_MEMORY = "memory";
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
public static final String BACKEND_NAME_SFTP = "sftp";
public static final String BACKEND_NAME_SFTP_WITH_VARIANTS = "sftpWithVariants";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String BACKEND_NAME_MEMORY = "memory";
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob";
public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_PERSON_SFTP = "person-sftp";
public static final String TABLE_NAME_BLOB_S3 = "s3-blob";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix";
public static final String TABLE_NAME_SFTP_FILE = "sftp-file";
public static final String TABLE_NAME_SFTP_FILE_VARIANTS = "sftp-file-with-variants";
public static final String TABLE_NAME_VARIANT_OPTIONS = "variant-options-table";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter";
@ -148,6 +161,7 @@ public class TestUtils
qInstance.addBackend(defineS3Backend());
qInstance.addBackend(defineS3BackendSansPrefix());
qInstance.addTable(defineS3CSVPersonTable());
qInstance.addTable(defineSFTPCSVPersonTable());
qInstance.addTable(defineS3BlobTable());
qInstance.addTable(defineS3BlobSansPrefixTable());
qInstance.addBackend(defineMockBackend());
@ -155,6 +169,15 @@ public class TestUtils
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
QBackendMetaData sftpBackend = defineSFTPBackend();
qInstance.addBackend(sftpBackend);
qInstance.addTable(defineSFTPFileTable(sftpBackend));
QBackendMetaData sftpBackendWithVariants = defineSFTPBackendWithVariants();
qInstance.addBackend(sftpBackendWithVariants);
qInstance.addTable(defineSFTPFileTableWithVariants(sftpBackendWithVariants));
qInstance.addTable(defineVariantOptionsTable());
definePersonCsvImporter(qInstance);
return (qInstance);
@ -162,6 +185,21 @@ public class TestUtils
/***************************************************************************
**
***************************************************************************/
private static QTableMetaData defineVariantOptionsTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_VARIANT_OPTIONS)
.withBackendName(defineMemoryBackend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("basePath", QFieldType.STRING));
}
/*******************************************************************************
**
*******************************************************************************/
@ -379,6 +417,25 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineSFTPCSVPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_SFTP)
.withLabel("Person SFTP Table")
.withBackendName(BACKEND_NAME_SFTP)
.withPrimaryKeyField("id")
.withFields(defineCommonPersonTableFields())
.withBackendDetails(new SFTPTableBackendDetails()
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
);
}
/*******************************************************************************
**
*******************************************************************************/
@ -463,4 +520,77 @@ public class TestUtils
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
return (mockAuthenticationModule.createSession(defineInstance(), null));
}
/***************************************************************************
**
***************************************************************************/
public static QTableMetaData defineSFTPFileTable(QBackendMetaData sftpBackend)
{
return new FilesystemTableMetaDataBuilder()
.withBasePath(BaseSFTPTest.TABLE_FOLDER)
.withBackend(sftpBackend)
.withName(TABLE_NAME_SFTP_FILE)
.buildStandardCardinalityOneTable()
.withLabel("SFTP Files");
}
/***************************************************************************
**
***************************************************************************/
private static QBackendMetaData defineSFTPBackend()
{
return (new SFTPBackendMetaData()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withHostName(BaseSFTPTest.HOST_NAME)
.withPort(BaseSFTPTest.getCurrentPort())
.withBasePath(BaseSFTPTest.BACKEND_FOLDER)
.withName(BACKEND_NAME_SFTP));
}
/***************************************************************************
**
***************************************************************************/
public static QTableMetaData defineSFTPFileTableWithVariants(QBackendMetaData sftpBackend)
{
return new FilesystemTableMetaDataBuilder()
.withBasePath(BaseSFTPTest.TABLE_FOLDER)
.withBackend(sftpBackend)
.withName(TABLE_NAME_SFTP_FILE_VARIANTS)
.buildStandardCardinalityOneTable()
.withLabel("SFTP Files");
}
/***************************************************************************
**
***************************************************************************/
private static QBackendMetaData defineSFTPBackendWithVariants()
{
return (new SFTPBackendMetaData()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withHostName(BaseSFTPTest.HOST_NAME)
.withPort(BaseSFTPTest.getCurrentPort())
////////////////////////////////////
// only get basePath from variant //
////////////////////////////////////
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withOptionsTableName(TABLE_NAME_VARIANT_OPTIONS)
.withVariantTypeKey(TABLE_NAME_VARIANT_OPTIONS)
.withBackendSettingSourceFieldNameMap(Map.of(
SFTPBackendVariantSetting.BASE_PATH, "basePath"
))
)
.withName(BACKEND_NAME_SFTP_WITH_VARIANTS));
}
}

154
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java Normal file
View File

@ -0,0 +1,154 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.MountableFile;
/*******************************************************************************
** Base class for tests that want to be able to work with sftp testcontainer
*******************************************************************************/
public class BaseSFTPTest extends BaseTest
{
public static final int PORT = 22;
public static final String USERNAME = "testuser";
public static final String PASSWORD = "testpass";
public static final String HOST_NAME = "localhost";
public static final String BACKEND_FOLDER = "upload";
public static final String TABLE_FOLDER = "files";
public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER;
private static GenericContainer<?> sftpContainer;
private static Integer currentPort;
/***************************************************************************
**
***************************************************************************/
@BeforeAll
static void setUp() throws Exception
{
sftpContainer = new GenericContainer<>("atmoz/sftp:latest")
.withExposedPorts(PORT)
.withCommand(USERNAME + ":" + PASSWORD + ":1001");
sftpContainer.start();
for(int i = 0; i < 5; i++)
{
copyFileToContainer("files/testfile.txt", REMOTE_DIR + "/testfile-" + i + ".txt");
}
grantUploadFilesDirWritePermission();
currentPort = sftpContainer.getMappedPort(22);
}
/***************************************************************************
**
***************************************************************************/
protected static void copyFileToContainer(String sourceFileClasspathResourceName, String fullRemotePath)
{
sftpContainer.copyFileToContainer(MountableFile.forClasspathResource(sourceFileClasspathResourceName), fullRemotePath);
}
/***************************************************************************
**
***************************************************************************/
protected static void rmrfInContainer(String fullRemotePath) throws Exception
{
sftpContainer.execInContainer("rm", "-rf", fullRemotePath);
}
/***************************************************************************
**
***************************************************************************/
@AfterAll
static void tearDown()
{
if(sftpContainer != null)
{
sftpContainer.stop();
}
}
/*******************************************************************************
** Getter for currentPort
**
*******************************************************************************/
public static Integer getCurrentPort()
{
return currentPort;
}
/***************************************************************************
**
***************************************************************************/
protected static void revokeUploadFilesDirWritePermission() throws Exception
{
setUploadFilesDirPermission("444");
}
/***************************************************************************
**
***************************************************************************/
protected static void grantUploadFilesDirWritePermission() throws Exception
{
setUploadFilesDirPermission("777");
}
/***************************************************************************
**
***************************************************************************/
private static void setUploadFilesDirPermission(String mode) throws Exception
{
sftpContainer.execInContainer("chmod", mode, "/home/testuser/upload/files");
}
/***************************************************************************
*
***************************************************************************/
protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception
{
sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path);
}
}

64
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java Normal file
View File

@ -0,0 +1,64 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPCountActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCount1() throws QException
{
CountInput countInput = initCountRequest();
SFTPCountAction countAction = new SFTPCountAction();
CountOutput countOutput = countAction.execute(countInput);
Assertions.assertEquals(5, countOutput.getCount(), "Expected # of rows from unfiltered count");
}
/*******************************************************************************
**
*******************************************************************************/
private CountInput initCountRequest() throws QException
{
CountInput countInput = new CountInput();
countInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE);
return countInput;
}
}

48
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java Normal file
View File

@ -0,0 +1,48 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPDeleteActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws QException
{
assertThrows(NotImplementedException.class, () -> new SFTPDeleteAction().execute(new DeleteInput()));
}
}

120
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java Normal file
View File

@ -0,0 +1,120 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPInsertActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityOne() throws QException, IOException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
SFTPInsertAction insertAction = new SFTPInsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseSFTPTest.BACKEND_FOLDER));
QRecord record = insertOutput.getRecords().get(0);
String fullPath = record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
assertThat(record.getErrors()).isNullOrEmpty();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityOnePermissionError() throws Exception
{
try
{
revokeUploadFilesDirWritePermission();
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
SFTPInsertAction insertAction = new SFTPInsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
QRecord record = insertOutput.getRecords().get(0);
assertThat(record.getErrors()).isNotEmpty();
assertThat(record.getErrors().get(0).getMessage()).contains("Error writing file: Permission denied");
}
finally
{
grantUploadFilesDirWritePermission();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityMany() throws QException, IOException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_SFTP);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")
));
SFTPInsertAction insertAction = new SFTPInsertAction();
assertThatThrownBy(() -> insertAction.execute(insertInput))
.hasRootCauseInstanceOf(NotImplementedException.class);
}
}

162
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java Normal file
View File

@ -0,0 +1,162 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Unit test for SFTPQueryAction
*******************************************************************************/
class SFTPQueryActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testSimpleQuery() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testQueryWithPath() throws Exception
{
String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
try
{
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE)
.withFilter(new QQueryFilter(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows from subfolder path query");
}
finally
{
rmrfInContainer(subfolderPath);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testQueryWithPathAndNameLike() throws Exception
{
String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
try
{
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
copyFileToContainer("files/testfile.txt", subfolderPath + "/who.txt");
Map<String, Integer> patternExpectedCountMap = Map.of(
"%.txt", 3,
"sub%", 2,
"%1%", 1,
"%", 3,
"*", 0
);
for(Map.Entry<String, Integer> entry : patternExpectedCountMap.entrySet())
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder"))
.withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey())));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey());
}
}
finally
{
rmrfInContainer(subfolderPath);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryVariantsTable() throws Exception
{
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_VARIANT_OPTIONS).withRecords(List.of(
new QRecord().withValue("id", 1).withValue("basePath", BaseSFTPTest.BACKEND_FOLDER),
new QRecord().withValue("id", 2).withValue("basePath", "empty-folder"),
new QRecord().withValue("id", 3).withValue("basePath", "non-existing-path")
)));
mkdirInSftpContainerUnderHomeTestuser("empty-folder/files");
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS);
assertThatThrownBy(() -> new QueryAction().execute(queryInput))
.hasMessageContaining("Could not find Backend Variant information in session under key 'variant-options-table' for Backend");
QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 2));
queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(0, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 3));
assertThatThrownBy(() -> new QueryAction().execute(queryInput))
.rootCause()
.hasMessageContaining("No such file");
// Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
}
}

115
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java Normal file
View File

@ -0,0 +1,115 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for FilesystemStorageAction
*******************************************************************************/
public class SFTPStorageActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testSmall() throws Exception
{
String data = "Hellooo, Storage.";
runTest(data);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testPermissionError() throws Exception
{
try
{
revokeUploadFilesDirWritePermission();
String data = "oops!";
assertThatThrownBy(() -> runTest(data))
.hasRootCauseInstanceOf(IOException.class)
.rootCause()
.hasMessageContaining("Permission denied");
}
finally
{
grantUploadFilesDirWritePermission();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testLarge() throws Exception
{
String data = StringUtils.join("!", Collections.nCopies(5_000_000, "Hello"));
runTest(data);
}
/***************************************************************************
**
***************************************************************************/
private static void runTest(String data) throws QException, IOException
{
StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_SFTP_FILE).withReference("fromStorageAction.txt");
StorageAction storageAction = new StorageAction();
OutputStream outputStream = storageAction.createOutputStream(storageInput);
outputStream.write(data.getBytes(StandardCharsets.UTF_8));
outputStream.close();
InputStream inputStream = storageAction.getInputStream(storageInput);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
inputStream.transferTo(byteArrayOutputStream);
assertEquals(data.length(), byteArrayOutputStream.toString(StandardCharsets.UTF_8).length());
assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8));
}
}

178
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java Normal file
View File

@ -0,0 +1,178 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for SFTPTestConnectionAction
*******************************************************************************/
class SFTPTestConnectionActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSuccessWithoutPath()
{
SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withPort(BaseSFTPTest.getCurrentPort())
.withHostName(BaseSFTPTest.HOST_NAME);
SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input);
assertTrue(output.getIsConnectionSuccess());
assertNull(output.getConnectionErrorMessage());
assertNull(output.getIsListBasePathSuccess());
assertNull(output.getListBasePathErrorMessage());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSuccessWithPath()
{
SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withPort(BaseSFTPTest.getCurrentPort())
.withHostName(BaseSFTPTest.HOST_NAME)
.withBasePath(BaseSFTPTest.BACKEND_FOLDER);
SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input);
assertTrue(output.getIsConnectionSuccess());
assertNull(output.getConnectionErrorMessage());
assertTrue(output.getIsListBasePathSuccess());
assertNull(output.getListBasePathErrorMessage());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSuccessfulConnectFailedPath()
{
SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withPort(BaseSFTPTest.getCurrentPort())
.withHostName(BaseSFTPTest.HOST_NAME)
.withBasePath("no-such-path");
SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input);
assertTrue(output.getIsConnectionSuccess());
assertNull(output.getConnectionErrorMessage());
assertFalse(output.getIsListBasePathSuccess());
assertNotNull(output.getListBasePathErrorMessage());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBadUsername()
{
SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput()
.withUsername("not-" + BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withPort(BaseSFTPTest.getCurrentPort())
.withHostName(BaseSFTPTest.HOST_NAME);
SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input);
assertFalse(output.getIsConnectionSuccess());
assertNotNull(output.getConnectionErrorMessage());
assertNull(output.getIsListBasePathSuccess());
assertNull(output.getListBasePathErrorMessage());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBadPassword()
{
SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword("not-" + BaseSFTPTest.PASSWORD)
.withPort(BaseSFTPTest.getCurrentPort())
.withHostName(BaseSFTPTest.HOST_NAME);
SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input);
assertFalse(output.getIsConnectionSuccess());
assertNotNull(output.getConnectionErrorMessage());
assertNull(output.getIsListBasePathSuccess());
assertNull(output.getListBasePathErrorMessage());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBadHostname()
{
SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withPort(BaseSFTPTest.getCurrentPort())
.withHostName("not-" + BaseSFTPTest.HOST_NAME);
SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input);
assertFalse(output.getIsConnectionSuccess());
assertNotNull(output.getConnectionErrorMessage());
assertNull(output.getIsListBasePathSuccess());
assertNull(output.getListBasePathErrorMessage());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBadPort()
{
SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withPort(10 * BaseSFTPTest.getCurrentPort())
.withHostName(BaseSFTPTest.HOST_NAME);
SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input);
assertFalse(output.getIsConnectionSuccess());
assertNotNull(output.getConnectionErrorMessage());
assertNull(output.getIsListBasePathSuccess());
assertNull(output.getListBasePathErrorMessage());
}
}

48
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java Normal file
View File

@ -0,0 +1,48 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPUpdateActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws QException
{
assertThrows(NotImplementedException.class, () -> new SFTPUpdateAction().execute(new UpdateInput()));
}
}

3
qqq-backend-module-filesystem/src/test/resources/files/testfile.txt Normal file
View File

@ -0,0 +1,3 @@
This is a file.
It is a test.

9
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
View File

@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
@ -132,6 +133,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
for(QRecord record : page)
{
QRecord outputRecord = new QRecord(record);
if(!StringUtils.hasContent(outputRecord.getTableName()))
{
outputRecord.setTableName(tableName);
}
outputRecords.add(outputRecord);
}
continue;
@ -151,6 +156,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
for(QRecord record : page)
{
QRecord outputRecord = new QRecord(record);
if(!StringUtils.hasContent(outputRecord.getTableName()))
{
outputRecord.setTableName(tableName);
}
outputRecords.add(outputRecord);
if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))

15
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -95,8 +95,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
@ -1200,15 +1198,18 @@ public class QJavalinImplementation
/////////////////////////////////////////////////////////////////////////////////////
if(backend != null && backend.getUsesVariants())
{
queryInput.setTableName(backend.getVariantOptionsTableName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backend.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backend.getVariantOptionsTableTypeValue())));
QTableMetaData variantsTable = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName());
queryInput.setTableName(variantsTable.getName());
queryInput.setFilter(backend.getBackendVariantsConfig().getOptionsFilter());
queryInput.setShouldGenerateDisplayValues(true);
QueryOutput output = new QueryAction().execute(queryInput);
for(QRecord qRecord : output.getRecords())
{
variants.add(new QFrontendVariant()
.withId(qRecord.getValue(backend.getVariantOptionsTableIdField()))
.withType(backend.getVariantOptionsTableTypeValue())
.withName(qRecord.getValueString(backend.getVariantOptionsTableNameField())));
.withId(qRecord.getValue(variantsTable.getPrimaryKeyField()))
.withType(backend.getBackendVariantsConfig().getVariantTypeKey())
.withName(qRecord.getRecordLabel()));
}
QJavalinAccessLogger.logStartSilent("variants");