QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 21:20:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged feature/sftp-and-headless-bulk-load into integration/sprint-61

Browse Source
This commit is contained in:
darin.kelkhoff
2025-02-20 09:30:07 -06:00
parent ead66385be be4f3c68f0
commit b1f3613e00
71 changed files with 5189 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java
View File

@ -815,13 +815,14 @@ public class RunProcessAction
{
QSession session = QContext.getQSession();
QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend());
if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey))
{
LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
}
else
{
basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
basepullKeyValue += "-" + session.getBackendVariants().get(variantTypeKey);
}
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java
View File

@ -82,6 +82,11 @@ public class DeleteAction
{
ActionHelper.validateSession(deleteInput);
if(deleteInput.getTableName() == null)
{
throw (new QException("Table name was not specified in delete input"));
}
QTableMetaData table = deleteInput.getTable();
String primaryKeyFieldName = table.getPrimaryKeyField();
QFieldMetaData primaryKeyField = table.getField(primaryKeyFieldName);

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -110,6 +111,12 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
public InsertOutput execute(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);
if(!StringUtils.hasContent(insertInput.getTableName()))
{
throw (new QException("Table name was not specified in update input"));
}
QTableMetaData table = insertInput.getTable();
if(table == null)

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java
View File

@ -47,7 +47,8 @@ public class StorageAction
{
/*******************************************************************************
**
** create an output stream in the storage backend - that can be written to,
** for the purpose of inserting or writing a file into storage.
*******************************************************************************/
public OutputStream createOutputStream(StorageInput storageInput) throws QException
{
@ -59,7 +60,8 @@ public class StorageAction
/*******************************************************************************
**
** create an input stream in the storage backend - that can be read from,
** for the purpose of getting or reading a file from storage.
*******************************************************************************/
public InputStream getInputStream(StorageInput storageInput) throws QException
{

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java
View File

@ -74,6 +74,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang.BooleanUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -118,6 +119,11 @@ public class UpdateAction
{
ActionHelper.validateSession(updateInput);
if(!StringUtils.hasContent(updateInput.getTableName()))
{
throw (new QException("Table name was not specified in update input"));
}
QTableMetaData table = updateInput.getTable();
//////////////////////////////////////////////////////

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
View File

@ -23,14 +23,18 @@ package com.kingsrook.qqq.backend.core.actions.values;
import java.io.Serializable;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
@ -508,7 +512,7 @@ public class QValueFormatter
{
@SuppressWarnings("unchecked") // instance validation should make this safe!
List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
List<String> values = CollectionUtils.nullSafeHasContents(fileNameFormatFields) ? fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList() : Collections.emptyList();
fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
}
}
@ -531,7 +535,7 @@ public class QValueFormatter
////////////////////////////////////////////////////////////////////////////////////////////////
// if field type is blob OR if there's a supplemental process or code-ref that needs to run - //
// then update its value to be a callback-url that'll give access to the bytes to download //
// then update its value to be a callback-url that'll give access to the bytes to download. //
// implied here is that a String value (w/o supplemental code/proc) has its value stay as a //
// URL, which is where the file is directly downloaded from. And in the case of a String //
// with code-to-run, then the code should run, followed by a redirect to the value URL. //
@ -540,7 +544,10 @@ public class QValueFormatter
|| adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE)
|| adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME))
{
record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
record.setValue(field.getName(), "/data/" + table.getName() + "/"
+ URLEncoder.encode(ValueUtils.getValueAsString(primaryKey), StandardCharsets.UTF_8) + "/"
+ field.getName() + "/"
+ URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8));
}
record.setDisplayValue(field.getName(), fileName);
}

98
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -37,7 +37,9 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TimeZone;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
@ -108,12 +110,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.Automatio
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ListingHash;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda;
import org.apache.commons.lang.BooleanUtils;
import org.quartz.CronExpression;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -543,6 +549,60 @@ public class QInstanceValidator
{
assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
///////////////////////
// validate variants //
///////////////////////
BackendVariantsConfig backendVariantsConfig = backend.getBackendVariantsConfig();
if(BooleanUtils.isTrue(backend.getUsesVariants()))
{
if(assertCondition(backendVariantsConfig != null, "Missing backendVariantsConfig in backend [" + backendName + "] which is marked as usesVariants"))
{
assertCondition(StringUtils.hasContent(backendVariantsConfig.getVariantTypeKey()), "Missing variantTypeKey in backendVariantsConfig in [" + backendName + "]");
String optionsTableName = backendVariantsConfig.getOptionsTableName();
QTableMetaData optionsTable = qInstance.getTable(optionsTableName);
if(assertCondition(StringUtils.hasContent(optionsTableName), "Missing optionsTableName in backendVariantsConfig in [" + backendName + "]"))
{
if(assertCondition(optionsTable != null, "Unrecognized optionsTableName [" + optionsTableName + "] in backendVariantsConfig in [" + backendName + "]"))
{
QQueryFilter optionsFilter = backendVariantsConfig.getOptionsFilter();
if(optionsFilter != null)
{
validateQueryFilter(qInstance, "optionsFilter in backendVariantsConfig in backend [" + backendName + "]: ", optionsTable, optionsFilter, null);
}
}
}
Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap = backendVariantsConfig.getBackendSettingSourceFieldNameMap();
if(assertCondition(CollectionUtils.nullSafeHasContents(backendSettingSourceFieldNameMap), "Missing or empty backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]"))
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
// only validate field names in the backendSettingSourceFieldNameMap if there is NOT a variantRecordSupplier //
// (the idea being, that the supplier might be building a record with fieldNames that aren't in the table... //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(optionsTable != null && backendVariantsConfig.getVariantRecordLookupFunction() == null)
{
for(Map.Entry<BackendVariantSetting, String> entry : backendSettingSourceFieldNameMap.entrySet())
{
assertCondition(optionsTable.getFields().containsKey(entry.getValue()), "Unrecognized fieldName [" + entry.getValue() + "] in backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]");
}
}
}
if(backendVariantsConfig.getVariantRecordLookupFunction() != null)
{
validateSimpleCodeReference("VariantRecordSupplier in backendVariantsConfig in backend [" + backendName + "]: ", backendVariantsConfig.getVariantRecordLookupFunction(), UnsafeFunction.class, Function.class);
}
}
}
else
{
assertCondition(backendVariantsConfig == null, "Should not have a backendVariantsConfig in backend [" + backendName + "] which is not marked as usesVariants");
}
///////////////////////////////////////////
// let the backend do its own validation //
///////////////////////////////////////////
backend.performValidation(this);
runPlugins(QBackendMetaData.class, backend, qInstance);
@ -1356,7 +1416,7 @@ public class QInstanceValidator
////////////////////////////////////////////////////////////////////////
if(customizerInstance != null && tableCustomizer.getExpectedType() != null)
{
assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance);
assertObjectCanBeCasted(prefix, customizerInstance, tableCustomizer.getExpectedType());
}
}
}
@ -1368,18 +1428,31 @@ public class QInstanceValidator
/*******************************************************************************
** Make sure that a given object can be casted to an expected type.
*******************************************************************************/
private <T> T assertObjectCanBeCasted(String errorPrefix, Class<T> expectedType, Object object)
private void assertObjectCanBeCasted(String errorPrefix, Object object, Class<?>... anyOfExpectedClasses)
{
T castedObject = null;
try
for(Class<?> expectedClass : anyOfExpectedClasses)
{
castedObject = expectedType.cast(object);
try
{
expectedClass.cast(object);
return;
}
catch(ClassCastException e)
{
/////////////////////////////////////
// try next type (if there is one) //
/////////////////////////////////////
}
}
catch(ClassCastException e)
if(anyOfExpectedClasses.length == 1)
{
errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType);
errors.add(errorPrefix + "CodeReference is not of the expected type: " + anyOfExpectedClasses[0]);
}
else
{
errors.add(errorPrefix + "CodeReference is not any of the expected types: " + Arrays.stream(anyOfExpectedClasses).map(c -> c.getName()).collect(Collectors.joining(", ")));
}
return castedObject;
}
@ -1616,12 +1689,12 @@ public class QInstanceValidator
for(QFieldMetaData field : process.getInputFields())
{
validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName());
validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName() + " ");
}
for(QFieldMetaData field : process.getOutputFields())
{
validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName());
validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName() + " ");
}
if(process.getCancelStep() != null)
@ -2123,7 +2196,8 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class<?> expectedClass)
@SafeVarargs
private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class<?>... anyOfExpectedClasses)
{
if(!preAssertionsForCodeReference(codeReference, prefix))
{
@ -2151,7 +2225,7 @@ public class QInstanceValidator
////////////////////////////////////////////////////////////////////////
if(classInstance != null)
{
assertObjectCanBeCasted(prefix, expectedClass, classInstance);
assertObjectCanBeCasted(prefix, classInstance, anyOfExpectedClasses);
}
}
}

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java
View File

@ -326,6 +326,20 @@ public class AuditSingleInput implements Serializable
/*******************************************************************************
** Fluent setter for details
*******************************************************************************/
public AuditSingleInput withDetailMessages(List<String> details)
{
for(String detail : details)
{
addDetail(message);
}
return (this);
}
/*******************************************************************************
**
*******************************************************************************/

25
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
import java.io.Serializable;
import java.time.Instant;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -34,9 +35,11 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback;
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface;
@ -247,6 +250,26 @@ public class RunBackendStepInput extends AbstractActionInput
/*******************************************************************************
** Getter for records converted to entities of a given type.
**
*******************************************************************************/
public <E extends QRecordEntity> List<E> getRecordsAsEntities(Class<E> entityClass) throws QException
{
List<E> rs = new ArrayList<>();
///////////////////////////////////////////////////////////////////////////////////
// note - important to call getRecords here, which is overwritten in subclasses! //
///////////////////////////////////////////////////////////////////////////////////
for(QRecord record : getRecords())
{
rs.add(QRecordEntity.fromQRecord(entityClass, record));
}
return (rs);
}
/*******************************************************************************
** Setter for records
**
@ -582,7 +605,7 @@ public class RunBackendStepInput extends AbstractActionInput
***************************************************************************/
public void traceMessage(ProcessTracerMessage message)
{
if(processTracer != null)
if(processTracer != null && message != null)
{
try
{

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java
View File

@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -258,7 +259,7 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
/*******************************************************************************
**
** add a record to the step output, e.g., for going through to the next step.
*******************************************************************************/
public void addRecord(QRecord record)
{
@ -271,6 +272,16 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
/***************************************************************************
** add a RecordEntity to the step output, e.g., for going through to the next step.
***************************************************************************/
public void addRecordEntity(QRecordEntity recordEntity)
{
addRecord(recordEntity.toQRecord());
}
/*******************************************************************************
** Getter for auditInputList
*******************************************************************************/

47
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java Normal file
View File

@ -0,0 +1,47 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata;
import com.kingsrook.qqq.backend.core.logging.QLogger;
/*******************************************************************************
** for use-cases where a metaDataProducer directly adds its objects to the
** qInstance, then this empty object can be returned.
*******************************************************************************/
public class EmptyMetaDataProducerOutput implements MetaDataProducerOutput
{
private static final QLogger LOG = QLogger.getLogger(EmptyMetaDataProducerOutput.class);
/***************************************************************************
**
***************************************************************************/
@Override
public void addSelfToInstance(QInstance instance)
{
/////////////////////////////////
// noop - this output is empty //
/////////////////////////////////
LOG.trace("empty meta data producer has nothing to add.");
}
}

235
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java
View File

@ -26,8 +26,13 @@ import java.util.HashSet;
import java.util.Set;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.serialization.QBackendMetaDataDeserializer;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@ -45,21 +50,18 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
private Set<Capability> enabledCapabilities = new HashSet<>();
private Set<Capability> disabledCapabilities = new HashSet<>();
private Boolean usesVariants = false;
private String variantOptionsTableIdField;
private String variantOptionsTableNameField;
private String variantOptionsTableTypeField;
private String variantOptionsTableTypeValue;
private String variantOptionsTableUsernameField;
private String variantOptionsTablePasswordField;
private String variantOptionsTableApiKeyField;
private String variantOptionsTableClientIdField;
private String variantOptionsTableClientSecretField;
private String variantOptionsTableName;
private Boolean usesVariants = false;
private BackendVariantsConfig backendVariantsConfig;
// todo - at some point, we may want to apply this to secret properties on subclasses?
// @JsonFilter("secretsFilter")
@Deprecated(since = "Replaced by filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
private String variantOptionsTableTypeField; // a field on which to filter the variant-options table, to limit which records in it are available as variants
@Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
private String variantOptionsTableTypeValue; // value for the type-field, to limit which records in it are available as variants; but also, the key in the session.backendVariants map!
/*******************************************************************************
@ -394,22 +396,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Getter for variantOptionsTableIdField
*******************************************************************************/
public String getVariantOptionsTableIdField()
{
return (this.variantOptionsTableIdField);
}
/*******************************************************************************
** Setter for variantOptionsTableIdField
*******************************************************************************/
@Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key")
public void setVariantOptionsTableIdField(String variantOptionsTableIdField)
{
this.variantOptionsTableIdField = variantOptionsTableIdField;
/////////////////////////////////////////////////
// noop as we migrate to backendVariantsConfig //
/////////////////////////////////////////////////
}
@ -417,30 +412,24 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableIdField
*******************************************************************************/
@Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key")
public QBackendMetaData withVariantOptionsTableIdField(String variantOptionsTableIdField)
{
this.variantOptionsTableIdField = variantOptionsTableIdField;
this.setVariantOptionsTableIdField(variantOptionsTableIdField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableNameField
*******************************************************************************/
public String getVariantOptionsTableNameField()
{
return (this.variantOptionsTableNameField);
}
/*******************************************************************************
** Setter for variantOptionsTableNameField
*******************************************************************************/
@Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel")
public void setVariantOptionsTableNameField(String variantOptionsTableNameField)
{
this.variantOptionsTableNameField = variantOptionsTableNameField;
/////////////////////////////////////////////////
// noop as we migrate to backendVariantsConfig //
/////////////////////////////////////////////////
}
@ -448,30 +437,26 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableNameField
*******************************************************************************/
@Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel")
public QBackendMetaData withVariantOptionsTableNameField(String variantOptionsTableNameField)
{
this.variantOptionsTableNameField = variantOptionsTableNameField;
this.setVariantOptionsTableNameField(variantOptionsTableNameField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableTypeField
*******************************************************************************/
public String getVariantOptionsTableTypeField()
{
return (this.variantOptionsTableTypeField);
}
/*******************************************************************************
** Setter for variantOptionsTableTypeField
*******************************************************************************/
@Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
public void setVariantOptionsTableTypeField(String variantOptionsTableTypeField)
{
this.variantOptionsTableTypeField = variantOptionsTableTypeField;
if(this.variantOptionsTableTypeValue != null)
{
this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue)));
}
}
@ -479,30 +464,28 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableTypeField
*******************************************************************************/
@Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
public QBackendMetaData withVariantOptionsTableTypeField(String variantOptionsTableTypeField)
{
this.variantOptionsTableTypeField = variantOptionsTableTypeField;
this.setVariantOptionsTableTypeField(variantOptionsTableTypeField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableTypeValue
*******************************************************************************/
public String getVariantOptionsTableTypeValue()
{
return (this.variantOptionsTableTypeValue);
}
/*******************************************************************************
** Setter for variantOptionsTableTypeValue
*******************************************************************************/
@Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
public void setVariantOptionsTableTypeValue(String variantOptionsTableTypeValue)
{
this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeValue);
this.variantOptionsTableTypeValue = variantOptionsTableTypeValue;
if(this.variantOptionsTableTypeField != null)
{
this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue)));
}
}
@ -510,30 +493,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableTypeValue
*******************************************************************************/
@Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
public QBackendMetaData withVariantOptionsTableTypeValue(String variantOptionsTableTypeValue)
{
this.variantOptionsTableTypeValue = variantOptionsTableTypeValue;
this.setVariantOptionsTableTypeValue(variantOptionsTableTypeValue);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableUsernameField
*******************************************************************************/
public String getVariantOptionsTableUsernameField()
{
return (this.variantOptionsTableUsernameField);
}
/*******************************************************************************
** Setter for variantOptionsTableUsernameField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTableUsernameField(String variantOptionsTableUsernameField)
{
this.variantOptionsTableUsernameField = variantOptionsTableUsernameField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.USERNAME, variantOptionsTableUsernameField);
}
@ -541,30 +516,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableUsernameField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTableUsernameField(String variantOptionsTableUsernameField)
{
this.variantOptionsTableUsernameField = variantOptionsTableUsernameField;
this.setVariantOptionsTableUsernameField(variantOptionsTableUsernameField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTablePasswordField
*******************************************************************************/
public String getVariantOptionsTablePasswordField()
{
return (this.variantOptionsTablePasswordField);
}
/*******************************************************************************
** Setter for variantOptionsTablePasswordField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTablePasswordField(String variantOptionsTablePasswordField)
{
this.variantOptionsTablePasswordField = variantOptionsTablePasswordField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.PASSWORD, variantOptionsTablePasswordField);
}
@ -572,30 +539,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTablePasswordField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTablePasswordField(String variantOptionsTablePasswordField)
{
this.variantOptionsTablePasswordField = variantOptionsTablePasswordField;
this.setVariantOptionsTablePasswordField(variantOptionsTablePasswordField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableApiKeyField
*******************************************************************************/
public String getVariantOptionsTableApiKeyField()
{
return (this.variantOptionsTableApiKeyField);
}
/*******************************************************************************
** Setter for variantOptionsTableApiKeyField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField)
{
this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.API_KEY, variantOptionsTableApiKeyField);
}
@ -603,30 +562,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableApiKeyField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField)
{
this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField;
this.setVariantOptionsTableApiKeyField(variantOptionsTableApiKeyField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableName
*******************************************************************************/
public String getVariantOptionsTableName()
{
return (this.variantOptionsTableName);
}
/*******************************************************************************
** Setter for variantOptionsTableName
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.tableName")
public void setVariantOptionsTableName(String variantOptionsTableName)
{
this.variantOptionsTableName = variantOptionsTableName;
this.getOrWithNewBackendVariantsConfig().withOptionsTableName(variantOptionsTableName);
}
@ -634,9 +585,10 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableName
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.tableName")
public QBackendMetaData withVariantOptionsTableName(String variantOptionsTableName)
{
this.variantOptionsTableName = variantOptionsTableName;
this.setVariantOptionsTableName(variantOptionsTableName);
return (this);
}
@ -651,22 +603,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
qInstance.addBackend(this);
}
/*******************************************************************************
** Getter for variantOptionsTableClientIdField
*******************************************************************************/
public String getVariantOptionsTableClientIdField()
{
return (this.variantOptionsTableClientIdField);
}
/*******************************************************************************
** Setter for variantOptionsTableClientIdField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
{
this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_ID, variantOptionsTableClientIdField);
}
@ -674,30 +619,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableClientIdField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
{
this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
this.setVariantOptionsTableClientIdField(variantOptionsTableClientIdField);
return (this);
}
/*******************************************************************************
** Getter for variantOptionsTableClientSecretField
*******************************************************************************/
public String getVariantOptionsTableClientSecretField()
{
return (this.variantOptionsTableClientSecretField);
}
/*******************************************************************************
** Setter for variantOptionsTableClientSecretField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
{
this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_SECRET, variantOptionsTableClientSecretField);
}
@ -705,11 +642,55 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
** Fluent setter for variantOptionsTableClientSecretField
*******************************************************************************/
@Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
{
this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
this.setVariantOptionsTableClientSecretField(variantOptionsTableClientSecretField);
return (this);
}
/*******************************************************************************
** Getter for backendVariantsConfig
*******************************************************************************/
public BackendVariantsConfig getBackendVariantsConfig()
{
return (this.backendVariantsConfig);
}
/*******************************************************************************
** Setter for backendVariantsConfig
*******************************************************************************/
public void setBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig)
{
this.backendVariantsConfig = backendVariantsConfig;
}
/*******************************************************************************
** Fluent setter for backendVariantsConfig
*******************************************************************************/
public QBackendMetaData withBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig)
{
this.backendVariantsConfig = backendVariantsConfig;
return (this);
}
/***************************************************************************
**
***************************************************************************/
private BackendVariantsConfig getOrWithNewBackendVariantsConfig()
{
if(backendVariantsConfig == null)
{
setBackendVariantsConfig(new BackendVariantsConfig());
}
return backendVariantsConfig;
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java
View File

@ -30,4 +30,5 @@ public enum AuditLevel
NONE,
RECORD,
FIELD
// idea: only audit changes to fields, e.g., on edit. though, is that a different dimension than this?
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java
View File

@ -86,7 +86,6 @@ public class QFrontendTableMetaData
//////////////////////////////////////////////////////////////////////////////////
/*******************************************************************************
**
*******************************************************************************/
@ -170,7 +169,7 @@ public class QFrontendTableMetaData
if(backend != null && backend.getUsesVariants())
{
usesVariants = true;
variantTableLabel = QContext.getQInstance().getTable(backend.getVariantOptionsTableName()).getLabel();
variantTableLabel = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName()).getLabel();
}
this.helpContents = tableMetaData.getHelpContent();

20
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java
View File

@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock
import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -712,6 +713,25 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
/*******************************************************************************
** Getter for sections
**
*******************************************************************************/
public QFieldSection getSection(String name)
{
for(QFieldSection qFieldSection : CollectionUtils.nonNullList(sections))
{
if(qFieldSection.getName().equals(name))
{
return (qFieldSection);
}
}
return (null);
}
/*******************************************************************************
** Setter for sections
**

221
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java Normal file
View File

@ -0,0 +1,221 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
/*******************************************************************************
** Factory class for creating "standard" qfield sections. e.g., if you want
** the same t1, t2, and t3 section on all your tables, use this class to
** produce them.
**
** You can change the default name & iconNames for those sections, but note,
** this is a static/utility style class, so those settings are static fields.
**
** The method customT2 is provided as not much of a shortcut over "doing it yourself",
** but to allow all sections for a table to be produced through calls to this factory,
** so they look more similar.
*******************************************************************************/
public class SectionFactory
{
private static String defaultT1name = "identity";
private static String defaultT1iconName = "badge";
private static String defaultT2name = "data";
private static String defaultT2iconName = "text_snippet";
private static String defaultT3name = "dates";
private static String defaultT3iconName = "calendar_month";
/*******************************************************************************
** private constructor, to enforce static usage, e.g., to make clear the fields
** are static fields.
**
*******************************************************************************/
private SectionFactory()
{
}
/***************************************************************************
**
***************************************************************************/
public static QFieldSection defaultT1(String... fieldNames)
{
return new QFieldSection(defaultT1name, new QIcon().withName(defaultT1iconName), Tier.T1, List.of(fieldNames));
}
/***************************************************************************
**
***************************************************************************/
public static QFieldSection defaultT2(String... fieldNames)
{
return new QFieldSection(defaultT2name, new QIcon().withName(defaultT2iconName), Tier.T2, List.of(fieldNames));
}
/***************************************************************************
**
***************************************************************************/
public static QFieldSection customT2(String name, QIcon icon, String... fieldNames)
{
return new QFieldSection(name, icon, Tier.T2, List.of(fieldNames));
}
/***************************************************************************
**
***************************************************************************/
public static QFieldSection defaultT3(String... fieldNames)
{
return new QFieldSection(defaultT3name, new QIcon().withName(defaultT3iconName), Tier.T3, List.of(fieldNames));
}
/*******************************************************************************
** Getter for defaultT1name
*******************************************************************************/
public static String getDefaultT1name()
{
return (SectionFactory.defaultT1name);
}
/*******************************************************************************
** Setter for defaultT1name
*******************************************************************************/
public static void setDefaultT1name(String defaultT1name)
{
SectionFactory.defaultT1name = defaultT1name;
}
/*******************************************************************************
** Getter for defaultT1iconName
*******************************************************************************/
public static String getDefaultT1iconName()
{
return (SectionFactory.defaultT1iconName);
}
/*******************************************************************************
** Setter for defaultT1iconName
*******************************************************************************/
public static void setDefaultT1iconName(String defaultT1iconName)
{
SectionFactory.defaultT1iconName = defaultT1iconName;
}
/*******************************************************************************
** Getter for defaultT2name
*******************************************************************************/
public static String getDefaultT2name()
{
return (SectionFactory.defaultT2name);
}
/*******************************************************************************
** Setter for defaultT2name
*******************************************************************************/
public static void setDefaultT2name(String defaultT2name)
{
SectionFactory.defaultT2name = defaultT2name;
}
/*******************************************************************************
** Getter for defaultT2iconName
*******************************************************************************/
public static String getDefaultT2iconName()
{
return (SectionFactory.defaultT2iconName);
}
/*******************************************************************************
** Setter for defaultT2iconName
*******************************************************************************/
public static void setDefaultT2iconName(String defaultT2iconName)
{
SectionFactory.defaultT2iconName = defaultT2iconName;
}
/*******************************************************************************
** Getter for defaultT3name
*******************************************************************************/
public static String getDefaultT3name()
{
return (SectionFactory.defaultT3name);
}
/*******************************************************************************
** Setter for defaultT3name
*******************************************************************************/
public static void setDefaultT3name(String defaultT3name)
{
SectionFactory.defaultT3name = defaultT3name;
}
/*******************************************************************************
** Getter for defaultT3iconName
*******************************************************************************/
public static String getDefaultT3iconName()
{
return (SectionFactory.defaultT3iconName);
}
/*******************************************************************************
** Setter for defaultT3iconName
*******************************************************************************/
public static void setDefaultT3iconName(String defaultT3iconName)
{
SectionFactory.defaultT3iconName = defaultT3iconName;
}
}

31
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java Normal file
View File

@ -0,0 +1,31 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
/*******************************************************************************
** interface to be implemented by enums (presumably) that define the possible
** settings a particular backend type can get from a variant record.
*******************************************************************************/
public interface BackendVariantSetting
{
}

225
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java Normal file
View File

@ -0,0 +1,225 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
import java.util.HashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
/*******************************************************************************
** Configs for how a backend that uses variants works. Specifically:
**
** - the variant "type key" - e.g., key for variants map in session.
** - what table supplies the variant options (optionsTableName
** - an optional filter to apply to that options table
** - a map of the settings that a backend gets from its variant table to the
** field names in that table that they come from. e.g., a backend may have a
** username attribute, whose value comes from a field named "theUser" in the
** variant options table.
** - an optional code reference to a variantRecordLookupFunction - to customize
** how the variant record is looked up (such as, adding joined or other custom
** fields).
*******************************************************************************/
public class BackendVariantsConfig
{
private String variantTypeKey;
private String optionsTableName;
private QQueryFilter optionsFilter;
private QCodeReference variantRecordLookupFunction;
private Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap;
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getOptionsTableName()
{
return (this.optionsTableName);
}
/*******************************************************************************
** Setter for tableName
*******************************************************************************/
public void setOptionsTableName(String optionsTableName)
{
this.optionsTableName = optionsTableName;
}
/*******************************************************************************
** Getter for filter
*******************************************************************************/
public QQueryFilter getOptionsFilter()
{
return (this.optionsFilter);
}
/*******************************************************************************
** Setter for filter
*******************************************************************************/
public void setOptionsFilter(QQueryFilter optionsFilter)
{
this.optionsFilter = optionsFilter;
}
/*******************************************************************************
** Getter for backendSettingSourceFieldNameMap
*******************************************************************************/
public Map<BackendVariantSetting, String> getBackendSettingSourceFieldNameMap()
{
return (this.backendSettingSourceFieldNameMap);
}
/*******************************************************************************
** Setter for backendSettingSourceFieldNameMap
*******************************************************************************/
public void setBackendSettingSourceFieldNameMap(Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap)
{
this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap;
}
/*******************************************************************************
** Fluent setter for backendSettingSourceFieldNameMap
*******************************************************************************/
public BackendVariantsConfig withBackendSettingSourceFieldName(BackendVariantSetting backendVariantSetting, String sourceFieldName)
{
if(this.backendSettingSourceFieldNameMap == null)
{
this.backendSettingSourceFieldNameMap = new HashMap<>();
}
this.backendSettingSourceFieldNameMap.put(backendVariantSetting, sourceFieldName);
return (this);
}
/*******************************************************************************
** Fluent setter for backendSettingSourceFieldNameMap
*******************************************************************************/
public BackendVariantsConfig withBackendSettingSourceFieldNameMap(Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap)
{
this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap;
return (this);
}
/*******************************************************************************
** Getter for variantTypeKey
*******************************************************************************/
public String getVariantTypeKey()
{
return (this.variantTypeKey);
}
/*******************************************************************************
** Setter for variantTypeKey
*******************************************************************************/
public void setVariantTypeKey(String variantTypeKey)
{
this.variantTypeKey = variantTypeKey;
}
/*******************************************************************************
** Fluent setter for variantTypeKey
*******************************************************************************/
public BackendVariantsConfig withVariantTypeKey(String variantTypeKey)
{
this.variantTypeKey = variantTypeKey;
return (this);
}
/*******************************************************************************
** Fluent setter for optionsTableName
*******************************************************************************/
public BackendVariantsConfig withOptionsTableName(String optionsTableName)
{
this.optionsTableName = optionsTableName;
return (this);
}
/*******************************************************************************
** Fluent setter for optionsFilter
*******************************************************************************/
public BackendVariantsConfig withOptionsFilter(QQueryFilter optionsFilter)
{
this.optionsFilter = optionsFilter;
return (this);
}
/*******************************************************************************
** Getter for variantRecordLookupFunction
*******************************************************************************/
public QCodeReference getVariantRecordLookupFunction()
{
return (this.variantRecordLookupFunction);
}
/*******************************************************************************
** Setter for variantRecordLookupFunction
*******************************************************************************/
public void setVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction)
{
this.variantRecordLookupFunction = variantRecordLookupFunction;
}
/*******************************************************************************
** Fluent setter for variantRecordLookupFunction
*******************************************************************************/
public BackendVariantsConfig withVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction)
{
this.variantRecordLookupFunction = variantRecordLookupFunction;
return (this);
}
}

106
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java Normal file
View File

@ -0,0 +1,106 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
import java.io.Serializable;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
/*******************************************************************************
** Utility methods for backends working with Variants.
*******************************************************************************/
public class BackendVariantsUtil
{
/*******************************************************************************
** Get the variant id from the session for the backend.
*******************************************************************************/
public static Serializable getVariantId(QBackendMetaData backendMetaData) throws QException
{
QSession session = QContext.getQSession();
String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey))
{
throw (new QException("Could not find Backend Variant information in session under key '" + variantTypeKey + "' for Backend '" + backendMetaData.getName() + "'"));
}
Serializable variantId = session.getBackendVariants().get(variantTypeKey);
return variantId;
}
/*******************************************************************************
** For backends that use variants, look up the variant record (in theory, based
** on an id in the session's backend variants map, then fetched from the backend's
** variant options table.
*******************************************************************************/
@SuppressWarnings("unchecked")
public static QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException
{
Serializable variantId = getVariantId(backendMetaData);
QRecord record;
if(backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction() != null)
{
Object o = QCodeLoader.getAdHoc(Object.class, backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction());
if(o instanceof UnsafeFunction<?,?,?> unsafeFunction)
{
record = ((UnsafeFunction<Serializable, QRecord, QException>) unsafeFunction).apply(variantId);
}
else if(o instanceof Function<?,?> function)
{
record = ((Function<Serializable, QRecord>) function).apply(variantId);
}
else
{
throw (new QException("Backend Variant's recordLookupFunction is not of any expected type (should have been caught by instance validation??)"));
}
}
else
{
GetInput getInput = new GetInput();
getInput.setShouldMaskPasswords(false);
getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
getInput.setPrimaryKey(variantId);
GetOutput getOutput = new GetAction().execute(getInput);
record = getOutput.getRecord();
}
if(record == null)
{
throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'"));
}
return record;
}
}

39
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java Normal file
View File

@ -0,0 +1,39 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
/*******************************************************************************
** temporary class, while we migrate from original way that variants were set up
** e.g., by calling 'variantOptionsTableUsernameField', to the new way, using
** the BackendVariantsConfig which uses a map of enum constants.
**
** so when those deprecated setters are removed, this enum can be too.
*****************************************************************************/
public enum LegacyBackendVariantSetting implements BackendVariantSetting
{
USERNAME,
PASSWORD,
API_KEY,
CLIENT_ID,
CLIENT_SECRET
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java
View File

@ -55,6 +55,8 @@ public class BulkInsertExtractStep extends AbstractExtractStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
int rowsAdded = 0;
int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);

82
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java
View File

@ -22,16 +22,38 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertLoadStep extends LoadViaInsertStep
public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSummaryProviderInterface
{
private static final QLogger LOG = QLogger.getLogger(BulkInsertLoadStep.class);
private Serializable firstInsertedPrimaryKey = null;
private Serializable lastInsertedPrimaryKey = null;
/*******************************************************************************
**
@ -42,4 +64,62 @@ public class BulkInsertLoadStep extends LoadViaInsertStep
return (QInputSource.USER);
}
/***************************************************************************
**
***************************************************************************/
@Override
public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
super.runOnePage(runBackendStepInput, runBackendStepOutput);
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName"));
List<QRecord> insertedRecords = runBackendStepOutput.getRecords();
for(QRecord insertedRecord : insertedRecords)
{
if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
{
if(firstInsertedPrimaryKey == null)
{
firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
}
lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
}
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
{
ArrayList<ProcessSummaryLineInterface> processSummary = getTransformStep().getProcessSummary(runBackendStepOutput, isForResultScreen);
try
{
if(firstInsertedPrimaryKey != null)
{
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepOutput.getValueString("tableName"));
QFieldMetaData field = table.getField(table.getPrimaryKeyField());
if(field.getType().isNumeric())
{
ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey);
idsLine.setCount(null);
processSummary.add(idsLine);
}
}
}
catch(Exception e)
{
LOG.warn("Error adding inserted-keys process summary line", e);
}
return (processSummary);
}
}

72
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java
View File

@ -32,16 +32,20 @@ import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
@ -63,12 +67,37 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
{
try
{
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
QRecord savedBulkLoadProfileRecord = BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
BulkLoadProfile bulkLoadProfile;
if(BulkInsertStepUtils.isHeadless(runBackendStepInput))
{
//////////////////////////////////////////////////////////////////////////////
// if running headless, build bulkLoadProfile from the saved profile record //
//////////////////////////////////////////////////////////////////////////////
if(savedBulkLoadProfileRecord == null)
{
throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load"));
}
SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
try
{
bulkLoadProfile = JsonUtils.toObject(savedBulkLoadProfile.getMappingJson(), BulkLoadProfile.class);
}
catch(Exception e)
{
throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e));
}
}
else
{
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
}
/////////////////////////////////////////////////////////////////////////
// put the list of bulk load profile into the process state - it's the //
@ -183,21 +212,32 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
/////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping))
if(BulkInsertStepUtils.isHeadless(runBackendStepInput))
{
//////////////////////////////////////////////////////////////////////////////////
// just go to the prepareValueMapping backend step - it'll figure out the rest. //
// it's also where the value-mapping loop of steps points. //
// and, this will actually be the default (e.g., the step after this one). //
//////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue("valueMappingFieldIndex", -1);
////////////////////////////////////////////////////////////////////////
// if running headless, always go straight to the preview screen next //
// todo actually, we could make this execute, right? //
////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
}
else
{
//////////////////////////////////////////////////////////////////////////////////
// else - if no values to map - continue with the standard streamed-ETL preview //
//////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping))
{
//////////////////////////////////////////////////////////////////////////////////
// just go to the prepareValueMapping backend step - it'll figure out the rest. //
// it's also where the value-mapping loop of steps points. //
// and, this will actually be the default (e.g., the step after this one). //
//////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue("valueMappingFieldIndex", -1);
}
else
{
//////////////////////////////////////////////////////////////////////////////////
// else - if no values to map - continue with the standard streamed-ETL preview //
//////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
}
}
}
catch(Exception e)

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java
View File

@ -29,11 +29,13 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerKeyRecordMessage;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.json.JSONArray;
@ -69,6 +71,18 @@ public class BulkInsertStepUtils
/***************************************************************************
**
***************************************************************************/
public static void setStorageInputForTheFile(RunProcessInput runProcessInput, StorageInput storageInput)
{
ArrayList<StorageInput> storageInputs = new ArrayList<>();
storageInputs.add(storageInput);
runProcessInput.addValue("theFile", storageInputs);
}
/***************************************************************************
**
***************************************************************************/
@ -144,13 +158,62 @@ public class BulkInsertStepUtils
/***************************************************************************
**
***************************************************************************/
public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
public static QRecord handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId");
if(savedBulkLoadProfileId != null)
{
QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId);
runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord);
return (savedBulkLoadProfileRecord);
}
return (null);
}
/***************************************************************************
**
***************************************************************************/
public static boolean isHeadless(RunBackendStepInput runBackendStepInput)
{
return (runBackendStepInput.getValuePrimitiveBoolean("isHeadless"));
}
/***************************************************************************
**
***************************************************************************/
public static void setHeadless(RunProcessInput runProcessInput)
{
runProcessInput.addValue("isHeadless", true);
}
/***************************************************************************
**
***************************************************************************/
public static void setProcessTracerKeyRecordMessage(RunProcessInput runProcessInput, ProcessTracerKeyRecordMessage processTracerKeyRecordMessage)
{
runProcessInput.addValue("processTracerKeyRecordMessage", processTracerKeyRecordMessage);
}
/***************************************************************************
**
***************************************************************************/
public static ProcessTracerKeyRecordMessage getProcessTracerKeyRecordMessage(RunBackendStepInput runBackendStepInput)
{
Serializable value = runBackendStepInput.getValue("processTracerKeyRecordMessage");
if(value instanceof ProcessTracerKeyRecordMessage processTracerKeyRecordMessage)
{
return (processTracerKeyRecordMessage);
}
return (null);
}
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java
View File

@ -258,6 +258,11 @@ public class BulkLoadValueMapper
valuesNotFound.add(value);
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// todo - we should probably be doing a lot of what QJavalinImplementation.finishPossibleValuesRequest does here //
// to apply possible-value filters. difficult to pass values in, but needed... //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
searchPossibleValueSourceInput.setIdList(idList);
searchPossibleValueSourceInput.setLimit(values.size());
LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0)));

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
View File

@ -239,6 +239,10 @@ public class ColumnStatsStep implements BackendStep
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, queryJoin == null ? null : List.of(queryJoin), null);
/////////////////////////////////////////////////////////////////////////////////////////////////
// todo - be aware of possible name collisions here!! (e.g., a table w/ a field named `count`) //
/////////////////////////////////////////////////////////////////////////////////////////////////
QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "count", countField), valueCounts);
runBackendStepOutput.addValue("valueCounts", valueCounts);

66
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java Normal file
View File

@ -0,0 +1,66 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.tracing;
/*******************************************************************************
** Specialization of process tracer message, to indicate a 'key record' that was
** used as an input or trigger to a process.
*******************************************************************************/
public class ProcessTracerKeyRecordMessage extends ProcessTracerMessage
{
private final String tableName;
private final Integer recordId;
/***************************************************************************
**
***************************************************************************/
public ProcessTracerKeyRecordMessage(String tableName, Integer recordId)
{
super("Process Key Record is " + tableName + " " + recordId);
this.tableName = tableName;
this.recordId = recordId;
}
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getTableName()
{
return (this.tableName);
}
/*******************************************************************************
** Getter for recordId
*******************************************************************************/
public Integer getRecordId()
{
return (this.recordId);
}
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java
View File

@ -22,13 +22,16 @@
package com.kingsrook.qqq.backend.core.processes.tracing;
import java.io.Serializable;
/*******************************************************************************
** Basic class that can be passed in to ProcessTracerInterface.handleMessage.
** This class just provides for a string message. We anticipate subclasses
** that may have more specific data, that specific tracer implementations may
** be aware of.
*******************************************************************************/
public class ProcessTracerMessage
public class ProcessTracerMessage implements Serializable
{
private String message;

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java
View File

@ -441,11 +441,16 @@ public class QScheduleManager
try
{
HashMap<String, Serializable> parameters = new HashMap<>(paramMap);
HashMap<String, Serializable> variantMap = new HashMap<>(Map.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())));
String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
String variantOptionsTableName = backendMetaData.getBackendVariantsConfig().getOptionsTableName();
String variantOptionsTableIdFieldName = QContext.getQInstance().getTable(variantOptionsTableName).getPrimaryKeyField();
HashMap<String, Serializable> variantMap = new HashMap<>(Map.of(variantTypeKey, qRecord.getValue(variantOptionsTableIdFieldName)));
parameters.put("backendVariantData", variantMap);
String identity = schedulableIdentity.getIdentity() + ";" + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField());
String description = schedulableIdentity.getDescription() + " for variant: " + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField());
String identity = schedulableIdentity.getIdentity() + ";" + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName);
String description = schedulableIdentity.getDescription() + " for variant: " + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName);
BasicSchedulableIdentity variantIdentity = new BasicSchedulableIdentity(identity, description);

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java
View File

@ -34,9 +34,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -102,7 +99,8 @@ public class SchedulerUtils
try
{
QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend());
Map<String, Serializable> thisVariantData = MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()));
QTableMetaData variantTable = QContext.getQInstance().getTable(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
Map<String, Serializable> thisVariantData = MapBuilder.of(backendMetaData.getBackendVariantsConfig().getVariantTypeKey(), qRecord.getValue(variantTable.getPrimaryKeyField()));
executeSingleProcess(process, thisVariantData, processInputValues);
}
catch(Exception e)
@ -181,8 +179,8 @@ public class SchedulerUtils
QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(processMetaData.getVariantBackend());
QueryInput queryInput = new QueryInput();
queryInput.setTableName(backendMetaData.getVariantOptionsTableName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue())));
queryInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
queryInput.setFilter(backendMetaData.getBackendVariantsConfig().getOptionsFilter());
QueryOutput queryOutput = new QueryAction().execute(queryInput);
records = queryOutput.getRecords();

144
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
View File

@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Consumer;
@ -55,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
@ -93,12 +95,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@ -182,6 +187,143 @@ public class QInstanceValidatorTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBackendVariants()
{
BackendVariantSetting setting = new BackendVariantSetting() {};
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)),
"Missing backendVariantsConfig in backend [variant] which is marked as usesVariants");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(false)
.withBackendVariantsConfig(new BackendVariantsConfig())),
"Should not have a backendVariantsConfig");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(null)
.withBackendVariantsConfig(new BackendVariantsConfig())),
"Should not have a backendVariantsConfig");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig())),
"Missing variantTypeKey in backendVariantsConfig",
"Missing optionsTableName in backendVariantsConfig",
"Missing or empty backendSettingSourceFieldNameMap");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName("notATable")
.withBackendSettingSourceFieldNameMap(Map.of(setting, "field")))),
"Unrecognized optionsTableName [notATable] in backendVariantsConfig");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withOptionsFilter(new QQueryFilter(new QFilterCriteria("notAField", QCriteriaOperator.EQUALS, 1)))
.withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")))),
"optionsFilter in backendVariantsConfig in backend [variant]: Criteria fieldName notAField is not a field");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withBackendSettingSourceFieldNameMap(Map.of(setting, "noSuchField")))),
"Unrecognized fieldName [noSuchField] in backendSettingSourceFieldNameMap");
assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withVariantRecordLookupFunction(new QCodeReference(CustomizerThatIsNotOfTheRightBaseClass.class))
.withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier"))
)),
"VariantRecordSupplier in backendVariantsConfig in backend [variant]: CodeReference is not any of the expected types: com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction, java.util.function.Function");
assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName"))
)));
assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withVariantRecordLookupFunction(new QCodeReference(VariantRecordFunction.class))
.withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier"))
)));
assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData()
.withName("variant")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withVariantTypeKey("myVariant")
.withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
.withVariantRecordLookupFunction(new QCodeReference(VariantRecordUnsafeFunction.class))
.withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier"))
)));
}
/***************************************************************************
**
***************************************************************************/
public static class VariantRecordFunction implements Function<Serializable, QRecord>
{
/***************************************************************************
**
***************************************************************************/
@Override
public QRecord apply(Serializable serializable)
{
return null;
}
}
/***************************************************************************
**
***************************************************************************/
public static class VariantRecordUnsafeFunction implements UnsafeFunction<Serializable, QRecord, QException>
{
/***************************************************************************
**
***************************************************************************/
@Override
public QRecord apply(Serializable serializable) throws QException
{
return null;
}
}
/*******************************************************************************
** Test an instance with null tables - should throw.
**
@ -2369,7 +2511,7 @@ public class QInstanceValidatorTest extends BaseTest
{
int noOfReasons = actualReasons == null ? 0 : actualReasons.size();
assertEquals(expectedReasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", expectedReasons)
+ "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--"));
+ "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--"));
}
for(String reason : expectedReasons)

44
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java Normal file
View File

@ -0,0 +1,44 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for EmptyMetaDataProducerOutput
*******************************************************************************/
class EmptyMetaDataProducerOutputTest
{
/*******************************************************************************
** sorry, just here to avoid a dip in coverage.
*******************************************************************************/
@Test
void test()
{
QInstance qInstance = new QInstance();
new EmptyMetaDataProducerOutput().addSelfToInstance(qInstance);
}
}

62
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java Normal file
View File

@ -0,0 +1,62 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for SectionFactory
*******************************************************************************/
class SectionFactoryTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test()
{
QFieldSection t1section = SectionFactory.defaultT1("id", "name");
assertEquals(SectionFactory.getDefaultT1name(), t1section.getName());
assertEquals(SectionFactory.getDefaultT1iconName(), t1section.getIcon().getName());
assertEquals(Tier.T1, t1section.getTier());
assertEquals(List.of("id", "name"), t1section.getFieldNames());
QFieldSection t2section = SectionFactory.defaultT2("size", "age");
assertEquals(SectionFactory.getDefaultT2name(), t2section.getName());
assertEquals(SectionFactory.getDefaultT2iconName(), t2section.getIcon().getName());
assertEquals(Tier.T2, t2section.getTier());
assertEquals(List.of("size", "age"), t2section.getFieldNames());
QFieldSection t3section = SectionFactory.defaultT3("createDate", "modifyDate");
assertEquals(SectionFactory.getDefaultT3name(), t3section.getName());
assertEquals(SectionFactory.getDefaultT3iconName(), t3section.getIcon().getName());
assertEquals(Tier.T3, t3section.getTier());
assertEquals(List.of("createDate", "modifyDate"), t3section.getFieldNames());
}
}

100
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java Normal file
View File

@ -0,0 +1,100 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.variants;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for BackendVariantsUtil
*******************************************************************************/
class BackendVariantsUtilTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetVariantId() throws QException
{
QBackendMetaData myBackend = getBackendMetaData();
assertThatThrownBy(() -> BackendVariantsUtil.getVariantId(myBackend))
.hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'");
QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701));
assertEquals(1701, BackendVariantsUtil.getVariantId(myBackend));
}
/***************************************************************************
**
***************************************************************************/
private static QBackendMetaData getBackendMetaData()
{
QBackendMetaData myBackend = new QBackendMetaData()
.withName("TestBackend")
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withOptionsTableName(TestUtils.TABLE_NAME_SHAPE)
.withVariantTypeKey("yourSelectedShape"));
return myBackend;
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetVariantRecord() throws QException
{
QBackendMetaData myBackend = getBackendMetaData();
TestUtils.insertDefaultShapes(QContext.getQInstance());
assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend))
.hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'");
QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701));
assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend))
.hasMessageContaining("Could not find Backend Variant in table shape with id '1701'");
QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1));
QRecord variantRecord = BackendVariantsUtil.getVariantRecord(myBackend);
assertEquals(1, variantRecord.getValueInteger("id"));
assertNotNull(variantRecord.getValue("name"));
}
}