QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 21:50:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feature/CE-779-order-level-ship-date

Browse Source
This commit is contained in:
Tim Chamberlain
2024-02-01 11:28:40 -06:00
parent 2681d66b32 612370fc13
commit 052cb9fab9
113 changed files with 10034 additions and 16826 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java
View File

@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
/*******************************************************************************
@ -30,12 +33,26 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
** part of a transaction.
**
** Most obvious use-case would be a JDBC Connection. See subclass in rdbms module.
** Ditto MongoDB.
**
** Note: One would imagine that this class shouldn't ever implement Serializable...
*******************************************************************************/
public class QBackendTransaction
{
/*******************************************************************************
**
*******************************************************************************/
public static QBackendTransaction openFor(AbstractTableActionInput input) throws QException
{
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(input.getBackend());
QBackendTransaction transaction = qModule.openTransaction(input);
return (transaction);
}
/*******************************************************************************
** Commit the transaction.
*******************************************************************************/

18
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java
View File

@ -30,10 +30,12 @@ import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
@ -73,6 +75,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
private static Set<String> loggedUnauditableTableNames = new HashSet<>();
/*******************************************************************************
@ -102,6 +105,21 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
return (output);
}
///////////////////////////////////////////////////////////////////////////////////////////////////////
// currently, the table's primary key must be integer... so, log (once) and return early if not that //
// (or, if no primary key!) //
///////////////////////////////////////////////////////////////////////////////////////////////////////
QFieldMetaData field = table.getFields().get(table.getPrimaryKeyField());
if(field == null || !QFieldType.INTEGER.equals(field.getType()))
{
if(!loggedUnauditableTableNames.contains(table.getName()))
{
LOG.info("Cannot audit table without integer as its primary key", logPair("tableName", table.getName()));
loggedUnauditableTableNames.add(table.getName());
}
return (output);
}
String contextSuffix = getContentSuffix(input);
AuditInput auditInput = new AuditInput();

127
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java
View File

@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
@ -87,8 +88,9 @@ public class PollingAutomationPerTableRunner implements Runnable
{
private static final QLogger LOG = QLogger.getLogger(PollingAutomationPerTableRunner.class);
private final TableActions tableActions;
private final String name;
private final TableActionsInterface tableActions;
private String name;
private QInstance instance;
private Supplier<QSession> sessionSupplier;
@ -116,10 +118,51 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
**
** Interface to be used by 2 records in this class - normal TableActions, and
** ShardedTableActions.
*******************************************************************************/
public record TableActions(String tableName, AutomationStatus status)
public interface TableActionsInterface
{
/*******************************************************************************
**
*******************************************************************************/
String tableName();
/*******************************************************************************
**
*******************************************************************************/
AutomationStatus status();
}
/*******************************************************************************
** Wrapper for a pair of (tableName, automationStatus)
*******************************************************************************/
public record TableActions(String tableName, AutomationStatus status) implements TableActionsInterface
{
/*******************************************************************************
**
*******************************************************************************/
public void noopToFakeTestCoverage()
{
}
}
/*******************************************************************************
** extended version of TableAction, for sharding use-case - adds the shard
** details.
*******************************************************************************/
public record ShardedTableActions(String tableName, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface
{
/*******************************************************************************
**
*******************************************************************************/
public void noopToFakeTestCoverage()
{
}
}
@ -128,16 +171,46 @@ public class PollingAutomationPerTableRunner implements Runnable
** basically just get a list of tables which at least *could* have automations
** run - either meta-data automations, or table-triggers (data/user defined).
*******************************************************************************/
public static List<TableActions> getTableActions(QInstance instance, String providerName)
public static List<TableActionsInterface> getTableActions(QInstance instance, String providerName)
{
List<TableActions> tableActionList = new ArrayList<>();
List<TableActionsInterface> tableActionList = new ArrayList<>();
for(QTableMetaData table : instance.getTables().values())
{
if(table.getAutomationDetails() != null && providerName.equals(table.getAutomationDetails().getProviderName()))
QTableAutomationDetails automationDetails = table.getAutomationDetails();
if(automationDetails != null && providerName.equals(automationDetails.getProviderName()))
{
tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
if(StringUtils.hasContent(automationDetails.getShardByFieldName()))
{
//////////////////////////////////////////////////////////////////////////////////////////////
// for sharded automations, add a tableAction (of the sharded subtype) for each shard-value //
//////////////////////////////////////////////////////////////////////////////////////////////
try
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(automationDetails.getShardSourceTableName());
QueryOutput queryOutput = new QueryAction().execute(queryInput);
for(QRecord record : queryOutput.getRecords())
{
Serializable shardId = record.getValue(automationDetails.getShardIdFieldName());
String label = record.getValueString(automationDetails.getShardLabelFieldName());
tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
}
}
catch(Exception e)
{
LOG.error("Error getting sharded table automation actions for a table", e, new LogPair("tableName", table.getName()));
}
}
else
{
///////////////////////////////////////////////////////////////////
// for non-sharded, we just need tabler name & automation status //
///////////////////////////////////////////////////////////////////
tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
}
}
}
@ -149,12 +222,17 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
**
*******************************************************************************/
public PollingAutomationPerTableRunner(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActions tableActions)
public PollingAutomationPerTableRunner(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActionsInterface tableActions)
{
this.instance = instance;
this.sessionSupplier = sessionSupplier;
this.tableActions = tableActions;
this.name = providerName + ">" + tableActions.tableName() + ">" + tableActions.status().getInsertOrUpdate();
if(tableActions instanceof ShardedTableActions shardedTableActions)
{
this.name += ":" + shardedTableActions.shardLabel();
}
}
@ -229,6 +307,15 @@ public class PollingAutomationPerTableRunner implements Runnable
throw (new NotImplementedException("Automation Status Tracking type [" + statusTrackingType + "] is not yet implemented in here."));
}
if(tableActions instanceof ShardedTableActions shardedTableActions)
{
//////////////////////////////////////////////////////////////
// for sharded actions, add the shardBy field as a criteria //
//////////////////////////////////////////////////////////////
QQueryFilter filter = queryInput.getFilter();
filter.addCriteria(new QFilterCriteria(shardedTableActions.shardByFieldName(), QCriteriaOperator.EQUALS, shardedTableActions.shardValue()));
}
queryInput.setRecordPipe(recordPipe);
return (new QueryAction().execute(queryInput));
}, () ->
@ -258,7 +345,23 @@ public class PollingAutomationPerTableRunner implements Runnable
{
if(action.getTriggerEvent().equals(triggerEvent))
{
rs.add(action);
///////////////////////////////////////////////////////////
// for sharded configs, only run if the shard id matches //
///////////////////////////////////////////////////////////
if(tableActions instanceof ShardedTableActions shardedTableActions)
{
if(shardedTableActions.shardValue().equals(action.getShardId()))
{
rs.add(action);
}
}
else
{
////////////////////////////////////////////
// for non-sharded, always add the action //
////////////////////////////////////////////
rs.add(action);
}
}
}
@ -471,7 +574,7 @@ public class PollingAutomationPerTableRunner implements Runnable
@Override
public QQueryFilter getQueryFilter()
{
List<Serializable> recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList());
List<Serializable> recordIds = records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).collect(Collectors.toList());
return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds)));
}
});

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java
View File

@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
** Interface for the Insert action.
**
*******************************************************************************/
public interface InsertInterface extends QActionInterface
public interface InsertInterface
{
/*******************************************************************************
**

44
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java
View File

@ -1,44 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.interfaces;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
/*******************************************************************************
**
*******************************************************************************/
public interface QActionInterface
{
/*******************************************************************************
**
*******************************************************************************/
default QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException
{
return (new QBackendTransaction());
}
}

16
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.metadata;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -150,14 +151,21 @@ public class MetaDataAction
}
metaDataOutput.setWidgets(widgets);
///////////////////////////////////////////////////////
// sort apps - by sortOrder (integer), then by label //
///////////////////////////////////////////////////////
List<QAppMetaData> sortedApps = metaDataInput.getInstance().getApps().values().stream()
.sorted(Comparator.comparing((QAppMetaData a) -> a.getSortOrder())
.thenComparing((QAppMetaData a) -> a.getLabel()))
.toList();
///////////////////////////////////
// map apps to frontend metadata //
///////////////////////////////////
Map<String, QFrontendAppMetaData> apps = new LinkedHashMap<>();
for(Map.Entry<String, QAppMetaData> entry : metaDataInput.getInstance().getApps().entrySet())
for(QAppMetaData app : sortedApps)
{
String appName = entry.getKey();
QAppMetaData app = entry.getValue();
String appName = app.getName();
PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, app);
if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
@ -191,7 +199,7 @@ public class MetaDataAction
// organize app tree nodes by their hierarchy //
////////////////////////////////////////////////
List<AppTreeNode> appTree = new ArrayList<>();
for(QAppMetaData appMetaData : metaDataInput.getInstance().getApps().values())
for(QAppMetaData appMetaData : sortedApps)
{
if(appMetaData.getParentAppName() == null)
{

68
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java Normal file
View File

@ -0,0 +1,68 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.processes;
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
/*******************************************************************************
** Constructor for commonly used QProcessCallback's
*******************************************************************************/
public class QProcessCallbackFactory
{
/*******************************************************************************
**
*******************************************************************************/
public static QProcessCallback forFilter(QQueryFilter filter)
{
return new QProcessCallback()
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public QQueryFilter getQueryFilter()
{
return (filter);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
{
return (Collections.emptyMap());
}
};
}
}

97
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import java.io.Serializable;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@ -52,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@ -116,21 +118,15 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
setAutomationStatusField(insertInput);
//////////////////////////////////////////////////////
// load the backend module and its insert interface //
//////////////////////////////////////////////////////
QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
InsertInterface insertInterface = qModule.getInsertInterface();
/////////////////////////////
// run standard validators //
/////////////////////////////
performValidations(insertInput, false);
////////////////////////////////////
// have the backend do the insert //
////////////////////////////////////
InsertOutput insertOutput = insertInterface.execute(insertInput);
//////////////////////////////////////////////////////
// use the backend module to actually do the insert //
//////////////////////////////////////////////////////
InsertOutput insertOutput = runInsertInBackend(insertInput);
if(insertOutput.getRecords() == null)
{
@ -195,6 +191,71 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
/*******************************************************************************
**
*******************************************************************************/
private InsertOutput runInsertInBackend(InsertInput insertInput) throws QException
{
///////////////////////////////////
// exit early if 0 input records //
///////////////////////////////////
if(CollectionUtils.nullSafeIsEmpty(insertInput.getRecords()))
{
LOG.debug("Insert request called with 0 records. Returning with no-op", logPair("tableName", insertInput.getTableName()));
InsertOutput rs = new InsertOutput();
rs.setRecords(new ArrayList<>());
return (rs);
}
/////////////////////////////////////////////
// set values in create date & modify date //
// todo .. better (not hard-coded names) //
/////////////////////////////////////////////
Instant now = Instant.now();
for(QRecord record : insertInput.getRecords())
{
setValueIfTableHasField(record, insertInput.getTable(), "createDate", now);
setValueIfTableHasField(record, insertInput.getTable(), "modifyDate", now);
}
//////////////////////////////////////////////////////
// load the backend module and its insert interface //
//////////////////////////////////////////////////////
QBackendModuleInterface qModule = getBackendModuleInterface(insertInput.getBackend());
InsertInterface insertInterface = qModule.getInsertInterface();
////////////////////////////////////
// have the backend do the insert //
////////////////////////////////////
InsertOutput insertOutput = insertInterface.execute(insertInput);
return insertOutput;
}
/*******************************************************************************
** If the table has a field with the given name, then set the given value in the
** given record.
*******************************************************************************/
private static void setValueIfTableHasField(QRecord record, QTableMetaData table, String fieldName, Serializable value)
{
try
{
if(table.getFields().containsKey(fieldName))
{
record.setValue(fieldName, value);
}
}
catch(Exception e)
{
/////////////////////////////////////////////////
// this means field doesn't exist, so, ignore. //
/////////////////////////////////////////////////
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -426,23 +487,11 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
/*******************************************************************************
**
*******************************************************************************/
private QBackendModuleInterface getBackendModuleInterface(InsertInput insertInput) throws QException
private QBackendModuleInterface getBackendModuleInterface(QBackendMetaData backend) throws QException
{
ActionHelper.validateSession(insertInput);
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend());
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend);
return (qModule);
}
/*******************************************************************************
**
*******************************************************************************/
public QBackendTransaction openTransaction(InsertInput insertInput) throws QException
{
QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
return (qModule.getInsertInterface().openTransaction(insertInput));
}
}

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java
View File

@ -84,9 +84,7 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
String primaryKeyField = table.getPrimaryKeyField();
if(transaction == null)
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(input.getTableName());
transaction = new InsertAction().openTransaction(insertInput);
transaction = QBackendTransaction.openFor(new InsertInput(input.getTableName()));
weOwnTheTransaction = true;
}

24
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java
View File

@ -135,7 +135,7 @@ public class UpdateAction
////////////////////////////////////
// have the backend do the update //
////////////////////////////////////
UpdateOutput updateOutput = updateInterface.execute(updateInput);
UpdateOutput updateOutput = runUpdateInBackend(updateInput, updateInterface);
if(updateOutput.getRecords() == null)
{
@ -203,6 +203,28 @@ public class UpdateAction
/*******************************************************************************
**
*******************************************************************************/
private UpdateOutput runUpdateInBackend(UpdateInput updateInput, UpdateInterface updateInterface) throws QException
{
///////////////////////////////////
// exit early if 0 input records //
///////////////////////////////////
if(CollectionUtils.nullSafeIsEmpty(updateInput.getRecords()))
{
LOG.debug("Update request called with 0 records. Returning with no-op", logPair("tableName", updateInput.getTableName()));
UpdateOutput rs = new UpdateOutput();
rs.setRecords(new ArrayList<>());
return (rs);
}
UpdateOutput updateOutput = updateInterface.execute(updateInput);
return updateOutput;
}
/*******************************************************************************
**
*******************************************************************************/

203
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UpdateActionRecordSplitHelper.java Normal file
View File

@ -0,0 +1,203 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers;
import java.io.Serializable;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ListingHash;
/*******************************************************************************
** Helper for backends that want to do their updates on records grouped by the
** set of fields that are being changed, and/or by the values those fields are
** being set to.
**
** e.g., RDBMS, for n records where some sub-set of fields are all having values
** set the same (say, a status=x), we can do that as 1 query where id in (?,?,...,?).
*******************************************************************************/
public class UpdateActionRecordSplitHelper
{
private ListingHash<List<String>, QRecord> recordsByFieldBeingUpdated = new ListingHash<>();
private boolean haveAnyWithoutErrors = false;
private List<QRecord> outputRecords = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
public void init(UpdateInput updateInput)
{
QTableMetaData table = updateInput.getTable();
Instant now = Instant.now();
for(QRecord record : updateInput.getRecords())
{
////////////////////////////////////////////
// todo .. better (not a hard-coded name) //
////////////////////////////////////////////
setValueIfTableHasField(record, table, "modifyDate", now);
List<String> updatableFields = table.getFields().values().stream()
.map(QFieldMetaData::getName)
// todo - intent here is to avoid non-updateable fields - but this
// should be like based on field.isUpdatable once that attribute exists
.filter(name -> !name.equals("id"))
.filter(name -> record.getValues().containsKey(name))
.toList();
recordsByFieldBeingUpdated.add(updatableFields, record);
if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
{
haveAnyWithoutErrors = true;
}
//////////////////////////////////////////////////////////////////////////////
// go ahead and put the record into the output list at this point in time, //
// so that the output list's order matches the input list order //
// note that if we want to capture updated values (like modify dates), then //
// we may want a map of primary key to output record, for easy updating. //
//////////////////////////////////////////////////////////////////////////////
QRecord outputRecord = new QRecord(record);
outputRecords.add(outputRecord);
}
}
/*******************************************************************************
**
*******************************************************************************/
public static boolean areAllValuesBeingUpdatedTheSame(UpdateInput updateInput, List<QRecord> recordList, List<String> fieldsBeingUpdated)
{
if(updateInput.getAreAllValuesBeingUpdatedTheSame() != null)
{
////////////////////////////////////////////////////////////
// if input told us what value to use here, then trust it //
////////////////////////////////////////////////////////////
return (updateInput.getAreAllValuesBeingUpdatedTheSame());
}
else
{
if(recordList.size() == 1)
{
//////////////////////////////////////////////////////
// if a single record, then yes, that always counts //
//////////////////////////////////////////////////////
return (true);
}
///////////////////////////////////////////////////////////////////////
// else iterate over the records, comparing them to the first record //
// return a false if any diffs are found. if no diffs, return true. //
///////////////////////////////////////////////////////////////////////
QRecord firstRecord = recordList.get(0);
for(int i = 1; i < recordList.size(); i++)
{
QRecord record = recordList.get(i);
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
///////////////////////////////////////////////////////
// skip records w/ errors (that we won't be updating //
///////////////////////////////////////////////////////
continue;
}
for(String fieldName : fieldsBeingUpdated)
{
if(!Objects.equals(firstRecord.getValue(fieldName), record.getValue(fieldName)))
{
return (false);
}
}
}
return (true);
}
}
/*******************************************************************************
** If the table has a field with the given name, then set the given value in the
** given record.
*******************************************************************************/
protected void setValueIfTableHasField(QRecord record, QTableMetaData table, String fieldName, Serializable value)
{
try
{
if(table.getFields().containsKey(fieldName))
{
record.setValue(fieldName, value);
}
}
catch(Exception e)
{
/////////////////////////////////////////////////
// this means field doesn't exist, so, ignore. //
/////////////////////////////////////////////////
}
}
/*******************************************************************************
** Getter for haveAnyWithoutErrors
**
*******************************************************************************/
public boolean getHaveAnyWithoutErrors()
{
return haveAnyWithoutErrors;
}
/*******************************************************************************
** Getter for recordsByFieldBeingUpdated
**
*******************************************************************************/
public ListingHash<List<String>, QRecord> getRecordsByFieldBeingUpdated()
{
return recordsByFieldBeingUpdated;
}
/*******************************************************************************
** Getter for outputRecords
**
*******************************************************************************/
public List<QRecord> getOutputRecords()
{
return outputRecords;
}
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java
View File

@ -34,7 +34,6 @@ import java.util.Objects;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
@ -104,7 +103,7 @@ public class QPossibleValueTranslator
{
if(!transactionsPerTable.containsKey(tableName))
{
transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName)));
transactionsPerTable.put(tableName, QBackendTransaction.openFor(new InsertInput(tableName)));
}
return (transactionsPerTable.get(tableName));

24
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
View File

@ -334,24 +334,34 @@ public class QValueFormatter
if(exposedJoin.getJoinTable().equals(nameParts[0]))
{
QTableMetaData joinTable = QContext.getQInstance().getTable(nameParts[0]);
fieldMap.put(fieldName, joinTable.getField(nameParts[1]));
if(joinTable.getFields().containsKey(nameParts[1]))
{
fieldMap.put(fieldName, joinTable.getField(nameParts[1]));
}
}
}
}
else
{
fieldMap.put(fieldName, table.getField(fieldName));
if(table.getFields().containsKey(fieldName))
{
fieldMap.put(fieldName, table.getField(fieldName));
}
}
}
catch(Exception e)
{
///////////////////////////////////////////////////////////
// put an empty field in - so no formatting will be done //
///////////////////////////////////////////////////////////
LOG.info("Error getting field for setting display value", e, logPair("fieldName", fieldName), logPair("tableName", table.getName()));
fieldMap.put(fieldName, new QFieldMetaData());
LOG.warn("Error getting field for setting display value", e, logPair("fieldName", fieldName), logPair("tableName", table.getName()));
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we didn't find the field definition, put an empty field in the map, so no formatting will be done //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
if(!fieldMap.containsKey(fieldName))
{
fieldMap.put(fieldName, new QFieldMetaData());
}
}
setDisplayValuesInRecord(fieldMap, record);

88
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java
View File

@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -133,11 +134,17 @@ public class CsvToQRecordAdapter
CSVFormat.DEFAULT
.withFirstRecordAsHeader()
.withIgnoreHeaderCase()
.withIgnoreEmptyLines()
.withTrim());
List<String> headers = csvParser.getHeaderNames();
headers = makeHeadersUnique(headers);
////////////////////////////////////////
// used by csv-headers-as-field-names //
////////////////////////////////////////
Map<String, QFieldMetaData> csvHeaderFieldMapping = buildCsvHeaderFieldMappingIfNeeded(inputWrapper, headers);
Iterator<CSVRecord> csvIterator = csvParser.iterator();
int recordCount = 0;
while(csvIterator.hasNext())
@ -160,11 +167,27 @@ public class CsvToQRecordAdapter
QRecord qRecord = new QRecord();
try
{
for(QFieldMetaData field : table.getFields().values())
if(inputWrapper.getCsvHeadersAsFieldNames())
{
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
/////////////////////////////////////////////////////////////////////////////////////////
// in csv-headers-as-field-names mode, don't mess with table, and don't do any mapping //
/////////////////////////////////////////////////////////////////////////////////////////
for(Map.Entry<String, String> entry : csvValues.entrySet())
{
setValue(inputWrapper, qRecord, csvHeaderFieldMapping.get(entry.getKey()), entry.getValue());
}
}
else
{
///////////////////////////////////////
// otherwise, fields come from table //
///////////////////////////////////////
for(QFieldMetaData field : table.getFields().values())
{
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
}
}
runRecordCustomizer(recordCustomizer, qRecord);
@ -247,6 +270,26 @@ public class CsvToQRecordAdapter
/*******************************************************************************
**
*******************************************************************************/
private Map<String, QFieldMetaData> buildCsvHeaderFieldMappingIfNeeded(InputWrapper inputWrapper, List<String> headers)
{
Map<String, QFieldMetaData> csvHeaderFieldMapping = null;
if(inputWrapper.getCsvHeadersAsFieldNames())
{
csvHeaderFieldMapping = new HashMap<>();
for(String header : headers)
{
header = adjustHeaderCase(header, inputWrapper);
csvHeaderFieldMapping.put(header, new QFieldMetaData(header, QFieldType.STRING));
}
}
return csvHeaderFieldMapping;
}
/*******************************************************************************
**
*******************************************************************************/
@ -376,7 +419,8 @@ public class CsvToQRecordAdapter
private Integer limit;
private boolean doCorrectValueTypes = false;
private boolean caseSensitiveHeaders = false;
private boolean caseSensitiveHeaders = false;
private boolean csvHeadersAsFieldNames = false;
@ -618,6 +662,40 @@ public class CsvToQRecordAdapter
/*******************************************************************************
** Getter for csvHeadersAsFieldNames
**
*******************************************************************************/
public boolean getCsvHeadersAsFieldNames()
{
return csvHeadersAsFieldNames;
}
/*******************************************************************************
** Setter for csvHeadersAsFieldNames
**
*******************************************************************************/
public void setCsvHeadersAsFieldNames(boolean csvHeadersAsFieldNames)
{
this.csvHeadersAsFieldNames = csvHeadersAsFieldNames;
}
/*******************************************************************************
** Fluent setter for csvHeadersAsFieldNames
**
*******************************************************************************/
public InputWrapper withCsvHeadersAsFieldNames(boolean csvHeadersAsFieldNames)
{
this.csvHeadersAsFieldNames = csvHeadersAsFieldNames;
return (this);
}
/*******************************************************************************
** Getter for doCorrectValueTypes
**

20
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapter.java
View File

@ -103,13 +103,23 @@ public class JsonToQRecordAdapter
{
QRecord record = new QRecord();
for(QFieldMetaData field : table.getFields().values())
if(table == null)
{
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
// todo - so if the mapping didn't say how to map this field, does that mean we should use the default name for the field?
if(jsonObject.has(fieldSource))
jsonObject.keys().forEachRemaining(key ->
{
record.setValue(field.getName(), (Serializable) jsonObject.get(fieldSource));
record.setValue(key, jsonObject.optString(key));
});
}
else
{
for(QFieldMetaData field : table.getFields().values())
{
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
// todo - so if the mapping didn't say how to map this field, does that mean we should use the default name for the field?
if(jsonObject.has(fieldSource))
{
record.setValue(field.getName(), (Serializable) jsonObject.get(fieldSource));
}
}
}

21
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -31,6 +31,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Stream;
@ -1244,7 +1245,25 @@ public class QInstanceValidator
{
if(fieldMetaData.getDefaultValue() != null && fieldMetaData.getDefaultValue() instanceof QCodeReference codeReference)
{
validateSimpleCodeReference("Process " + processName + " backend step code reference: ", codeReference, BackendStep.class);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// by default, assume that any process field which is a QCodeReference should be a reference to a BackendStep... //
// but... allow a secondary field name to be set, to tell us what class to *actually* expect here... //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Class<?> expectedClass = BackendStep.class;
try
{
Optional<QFieldMetaData> expectedTypeField = backendStepMetaData.getInputMetaData().getField(fieldMetaData.getName() + "_expectedType");
if(expectedTypeField.isPresent() && expectedTypeField.get().getDefaultValue() != null)
{
expectedClass = Class.forName(ValueUtils.getValueAsString(expectedTypeField.get().getDefaultValue()));
}
}
catch(Exception e)
{
warn("Error loading expectedType for field [" + fieldMetaData.getName() + "] in process [" + processName + "]: " + e.getMessage());
}
validateSimpleCodeReference("Process " + processName + " code reference: ", codeReference, expectedClass);
}
}
}

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.kingsrook.qqq.backend.core.logging.QLogger;
@ -42,7 +43,7 @@ public class QFilterCriteria implements Serializable, Cloneable
{
private static final QLogger LOG = QLogger.getLogger(QFilterCriteria.class);
private String fieldName;
private String fieldName;
private QCriteriaOperator operator;
private List<Serializable> values;
@ -93,11 +94,37 @@ public class QFilterCriteria implements Serializable, Cloneable
/*******************************************************************************
**
*******************************************************************************/
public QFilterCriteria(String fieldName, QCriteriaOperator operator, List<Serializable> values)
@SuppressWarnings("unchecked")
public QFilterCriteria(String fieldName, QCriteriaOperator operator, List<? extends Serializable> values)
{
this.fieldName = fieldName;
this.operator = operator;
this.values = values == null ? new ArrayList<>() : values;
this.values = values == null ? new ArrayList<>() : (List<Serializable>) values;
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings({ "rawtypes", "unchecked" })
public QFilterCriteria(String fieldName, QCriteriaOperator operator, Collection<? extends Serializable> values)
{
this.fieldName = fieldName;
this.operator = operator;
if(values == null)
{
this.values = new ArrayList<>();
}
else if(values instanceof List list)
{
this.values = list;
}
else
{
this.values = new ArrayList<>(values);
}
}

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java
View File

@ -167,6 +167,16 @@ public class QRecord implements Serializable
ArrayList<?> cloneList = new ArrayList<>(arrayList);
clone.put(entry.getKey(), (V) cloneList);
}
else if(entry.getValue() instanceof LinkedHashMap<?, ?> linkedHashMap)
{
LinkedHashMap<?, ?> cloneMap = new LinkedHashMap<>(linkedHashMap);
clone.put(entry.getKey(), (V) cloneMap);
}
else if(entry.getValue() instanceof HashMap<?, ?> hashMap)
{
HashMap<?, ?> cloneMap = new HashMap<>(hashMap);
clone.put(entry.getKey(), (V) cloneMap);
}
else if(entry.getValue() instanceof QRecord otherQRecord)
{
clone.put(entry.getKey(), (V) new QRecord(otherQRecord));

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java
View File

@ -95,6 +95,7 @@ public class MetaDataProducerHelper
////////////////////////////////////////////////////////////////////////////////////////////
// sort them by sort order, then by the type that they return - specifically - doing apps //
// after all other types (as apps often try to get other types from the instance) //
// also - do backends earlier than others (e.g., tables may expect backends to exist) //
////////////////////////////////////////////////////////////////////////////////////////////
producers.sort(Comparator
.comparing((MetaDataProducerInterface<?> p) -> p.getSortOrder())
@ -105,11 +106,15 @@ public class MetaDataProducerHelper
Class<?> outputType = p.getClass().getMethod("produce", QInstance.class).getReturnType();
if(outputType.equals(QAppMetaData.class))
{
return (1);
return (2);
}
else if(outputType.equals(QBackendMetaData.class))
{
return (0);
}
else
{
return (0);
return (1);
}
}
catch(Exception e)

24
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java
View File

@ -746,12 +746,22 @@ public class QInstance
/*******************************************************************************
** Setter for hasBeenValidated
** If pass a QInstanceValidationKey (which can only be instantiated by the validator),
** then the hasBeenValidated field will be set to true.
**
** Else, if passed a null, hasBeenValidated will be reset to false - e.g., to
** re-trigger validation (can be useful in tests).
*******************************************************************************/
public void setHasBeenValidated(QInstanceValidationKey key)
{
this.hasBeenValidated = true;
if(key == null)
{
this.hasBeenValidated = false;
}
else
{
this.hasBeenValidated = true;
}
}
@ -1198,4 +1208,14 @@ public class QInstance
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public void add(TopLevelMetaDataInterface metaData)
{
metaData.addSelfToInstance(this);
}
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java
View File

@ -43,6 +43,8 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu
private String name;
private String label;
private Integer sortOrder = 500;
private QPermissionRules permissionRules;
private List<QAppChildMetaData> children;
@ -426,4 +428,36 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu
{
qInstance.addApp(this);
}
/*******************************************************************************
** Getter for sortOrder
*******************************************************************************/
public Integer getSortOrder()
{
return (this.sortOrder);
}
/*******************************************************************************
** Setter for sortOrder
*******************************************************************************/
public void setSortOrder(Integer sortOrder)
{
this.sortOrder = sortOrder;
}
/*******************************************************************************
** Fluent setter for sortOrder
*******************************************************************************/
public QAppMetaData withSortOrder(Integer sortOrder)
{
this.sortOrder = sortOrder;
return (this);
}
}

56
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java
View File

@ -23,8 +23,11 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -32,6 +35,8 @@ import com.kingsrook.qqq.backend.core.processes.implementations.basepull.Basepul
*******************************************************************************/
public class AbstractProcessMetaDataBuilder
{
private static final QLogger LOG = QLogger.getLogger(AbstractProcessMetaDataBuilder.class);
protected QProcessMetaData processMetaData;
@ -58,6 +63,54 @@ public class AbstractProcessMetaDataBuilder
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public AbstractProcessMetaDataBuilder withName(String name)
{
processMetaData.setName(name);
return (this);
}
/*******************************************************************************
** Fluent setter for label
**
*******************************************************************************/
public AbstractProcessMetaDataBuilder withLabel(String name)
{
processMetaData.setLabel(name);
return (this);
}
/*******************************************************************************
** Fluent setter for tableName
**
*******************************************************************************/
public AbstractProcessMetaDataBuilder withTableName(String tableName)
{
processMetaData.setTableName(tableName);
return (this);
}
/*******************************************************************************
** Fluent setter for icon
**
*******************************************************************************/
public AbstractProcessMetaDataBuilder withIcon(QIcon icon)
{
processMetaData.setIcon(icon);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@ -65,7 +118,8 @@ public class AbstractProcessMetaDataBuilder
{
processMetaData.getInputFields().stream()
.filter(f -> f.getName().equals(fieldName)).findFirst()
.ifPresent(f -> f.setDefaultValue(value));
.ifPresentOrElse(f -> f.setDefaultValue(value),
() -> LOG.warn("Could not find process input field for setting default value", logPair("processName", () -> processMetaData.getName()), logPair("fieldName", fieldName)));
}

22
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java
View File

@ -34,6 +34,7 @@ import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntityField;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@ -49,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRule
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -57,6 +59,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
*******************************************************************************/
public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface
{
private static final QLogger LOG = QLogger.getLogger(QTableMetaData.class);
private String name;
private String label;
@ -813,6 +817,15 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
*******************************************************************************/
public QTableMetaData withUniqueKey(UniqueKey uniqueKey)
{
////////////////////////////////////////////////////////////////////////////////////
// you can't add a null key, so, if someone tried, just gracefully return w/ noop //
////////////////////////////////////////////////////////////////////////////////////
if(uniqueKey == null)
{
LOG.debug("Skipping request to add null uniqueKey", logPair("tableName", name));
return (this);
}
if(this.uniqueKeys == null)
{
this.uniqueKeys = new ArrayList<>();
@ -1130,6 +1143,15 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
*******************************************************************************/
public QTableMetaData withRecordSecurityLock(RecordSecurityLock recordSecurityLock)
{
/////////////////////////////////////////////////////////////////////////////////////
// you can't add a null lock, so, if someone tried, just gracefully return w/ noop //
/////////////////////////////////////////////////////////////////////////////////////
if(recordSecurityLock == null)
{
LOG.debug("Skipping request to add null recordSecurityLock", logPair("tableName", name));
return (this);
}
if(this.recordSecurityLocks == null)
{
this.recordSecurityLocks = new ArrayList<>();

129
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java
View File

@ -37,6 +37,11 @@ public class QTableAutomationDetails
private Integer overrideBatchSize;
private String shardByFieldName; // field in "this" table, to use for sharding
private String shardSourceTableName; // name of the table where the shards are defined as rows
private String shardLabelFieldName; // field in shard-source-table to use for labeling shards
private String shardIdFieldName; // field in shard-source-table to identify shards (e.g., joins to this table's shardByFieldName)
/*******************************************************************************
@ -188,4 +193,128 @@ public class QTableAutomationDetails
return (this);
}
/*******************************************************************************
** Getter for shardByFieldName
*******************************************************************************/
public String getShardByFieldName()
{
return (this.shardByFieldName);
}
/*******************************************************************************
** Setter for shardByFieldName
*******************************************************************************/
public void setShardByFieldName(String shardByFieldName)
{
this.shardByFieldName = shardByFieldName;
}
/*******************************************************************************
** Fluent setter for shardByFieldName
*******************************************************************************/
public QTableAutomationDetails withShardByFieldName(String shardByFieldName)
{
this.shardByFieldName = shardByFieldName;
return (this);
}
/*******************************************************************************
** Getter for shardSourceTableName
*******************************************************************************/
public String getShardSourceTableName()
{
return (this.shardSourceTableName);
}
/*******************************************************************************
** Setter for shardSourceTableName
*******************************************************************************/
public void setShardSourceTableName(String shardSourceTableName)
{
this.shardSourceTableName = shardSourceTableName;
}
/*******************************************************************************
** Fluent setter for shardSourceTableName
*******************************************************************************/
public QTableAutomationDetails withShardSourceTableName(String shardSourceTableName)
{
this.shardSourceTableName = shardSourceTableName;
return (this);
}
/*******************************************************************************
** Getter for shardLabelFieldName
*******************************************************************************/
public String getShardLabelFieldName()
{
return (this.shardLabelFieldName);
}
/*******************************************************************************
** Setter for shardLabelFieldName
*******************************************************************************/
public void setShardLabelFieldName(String shardLabelFieldName)
{
this.shardLabelFieldName = shardLabelFieldName;
}
/*******************************************************************************
** Fluent setter for shardLabelFieldName
*******************************************************************************/
public QTableAutomationDetails withShardLabelFieldName(String shardLabelFieldName)
{
this.shardLabelFieldName = shardLabelFieldName;
return (this);
}
/*******************************************************************************
** Getter for shardIdFieldName
*******************************************************************************/
public String getShardIdFieldName()
{
return (this.shardIdFieldName);
}
/*******************************************************************************
** Setter for shardIdFieldName
*******************************************************************************/
public void setShardIdFieldName(String shardIdFieldName)
{
this.shardIdFieldName = shardIdFieldName;
}
/*******************************************************************************
** Fluent setter for shardIdFieldName
*******************************************************************************/
public QTableAutomationDetails withShardIdFieldName(String shardIdFieldName)
{
this.shardIdFieldName = shardIdFieldName;
return (this);
}
}

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java
View File

@ -37,6 +37,7 @@ public class TableAutomationAction
private TriggerEvent triggerEvent;
private Integer priority = 500;
private QQueryFilter filter;
private Serializable shardId;
////////////////////////////////////////////////////////////////////////
// flag that will cause the records to cause their associations to be //
@ -329,4 +330,35 @@ public class TableAutomationAction
return (this);
}
/*******************************************************************************
** Getter for shardId
*******************************************************************************/
public Serializable getShardId()
{
return (this.shardId);
}
/*******************************************************************************
** Setter for shardId
*******************************************************************************/
public void setShardId(Serializable shardId)
{
this.shardId = shardId;
}
/*******************************************************************************
** Fluent setter for shardId
*******************************************************************************/
public TableAutomationAction withShardId(Serializable shardId)
{
this.shardId = shardId;
return (this);
}
}

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.modules.backend;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
@ -29,6 +30,8 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
@ -126,6 +129,14 @@ public interface QBackendModuleInterface
return null;
}
/*******************************************************************************
**
*******************************************************************************/
default QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException
{
return (new QBackendTransaction());
}
/*******************************************************************************
**
*******************************************************************************/

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/AbstractMemoryAction.java
View File

@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.actions.interfaces.QActionInterface;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -32,7 +31,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Base class for all core actions in the Memory backend module.
*******************************************************************************/
public abstract class AbstractMemoryAction implements QActionInterface
public abstract class AbstractMemoryAction
{
/*******************************************************************************

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryInsertAction.java
View File

@ -22,13 +22,10 @@
package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
@ -45,18 +42,6 @@ public class MemoryInsertAction extends AbstractMemoryAction implements InsertIn
{
try
{
QTableMetaData table = insertInput.getTable();
Instant now = Instant.now();
for(QRecord record : insertInput.getRecords())
{
///////////////////////////////////////////
// todo .. better (not hard-coded names) //
///////////////////////////////////////////
setValueIfTableHasField(record, table, "createDate", now, false);
setValueIfTableHasField(record, table, "modifyDate", now, false);
}
InsertOutput insertOutput = new InsertOutput();
insertOutput.setRecords(MemoryRecordStore.getInstance().insert(insertInput, true));
return (insertOutput);

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java
View File

@ -27,7 +27,6 @@ import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@ -116,7 +115,7 @@ public class StreamedETLBackendStep implements BackendStep
insertInput.setTableName(runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE));
return new InsertAction().openTransaction(insertInput);
return QBackendTransaction.openFor(insertInput);
}

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java
View File

@ -26,7 +26,6 @@ import java.util.Optional;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@ -34,7 +33,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -87,9 +85,8 @@ public class LoadViaDeleteStep extends AbstractLoadStep
@Override
public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(new InsertAction().openTransaction(insertInput)));
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(QBackendTransaction.openFor(deleteInput)));
}
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java
View File

@ -129,8 +129,7 @@ public class LoadViaInsertOrUpdateStep extends AbstractLoadStep
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(new InsertAction().openTransaction(insertInput)));
return (Optional.of(QBackendTransaction.openFor(insertInput)));
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java
View File

@ -88,7 +88,6 @@ public class LoadViaInsertStep extends AbstractLoadStep
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(new InsertAction().openTransaction(insertInput)));
return (Optional.of(QBackendTransaction.openFor(insertInput)));
}
}

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java
View File

@ -24,14 +24,12 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
@ -81,9 +79,8 @@ public class LoadViaUpdateStep extends AbstractLoadStep
@Override
public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(new InsertAction().openTransaction(insertInput)));
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(QBackendTransaction.openFor(updateInput)));
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
View File

@ -34,7 +34,6 @@ import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java
View File

@ -75,7 +75,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
InsertAction insertAction = new InsertAction();
InsertInput insertInput = new InsertInput();
insertInput.setTableName(ScriptRevision.TABLE_NAME);
QBackendTransaction transaction = insertAction.openTransaction(insertInput);
QBackendTransaction transaction = QBackendTransaction.openFor(insertInput);
insertInput.setTransaction(transaction);
try

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java
View File

@ -182,6 +182,14 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
public record SyncProcessConfig(String sourceTable, String sourceTableKeyField, String destinationTable, String destinationTableForeignKey, boolean performInserts, boolean performUpdates)
{
/*******************************************************************************
** Overloaded constructor - defaults both performInserts & performUpdates to true.
*******************************************************************************/
public SyncProcessConfig(String sourceTable, String sourceTableKeyField, String destinationTable, String destinationTableForeignKey)
{
this(sourceTable, sourceTableKeyField, destinationTable, destinationTableForeignKey, true, true);
}
/*******************************************************************************
** artificial method, here to make jacoco see that this class is indeed
** included in test coverage...
@ -268,6 +276,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
if(sourceKeyValue == null || "".equals(sourceKeyValue))
{
LOG.debug("Skipping record without a value in the sourceKeyField", logPair("keyField", sourceTableKeyField));
errorMissingKeyField.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
try

96
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java
View File

@ -120,52 +120,68 @@ public class ScheduleManager
return;
}
for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values())
boolean needToClearContext = false;
try
{
startQueueProvider(queueProvider);
}
for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values())
{
startAutomationProviderPerTable(automationProvider);
}
for(QProcessMetaData process : qInstance.getProcesses().values())
{
if(process.getSchedule() != null && allowedToStart(process.getName()))
if(QContext.getQInstance() == null)
{
QScheduleMetaData scheduleMetaData = process.getSchedule();
if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy()))
needToClearContext = true;
QContext.init(qInstance, sessionSupplier.get());
}
for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values())
{
startQueueProvider(queueProvider);
}
for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values())
{
startAutomationProviderPerTable(automationProvider);
}
for(QProcessMetaData process : qInstance.getProcesses().values())
{
if(process.getSchedule() != null && allowedToStart(process.getName()))
{
///////////////////////////////////////////////
// if no variants, or variant is serial mode //
///////////////////////////////////////////////
startProcess(process, null);
}
else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy()))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////
// if this a "parallel", which for example means we want to have a thread for each backend variant //
// running at the same time, get the variant records and schedule each separately //
/////////////////////////////////////////////////////////////////////////////////////////////////////
QContext.init(qInstance, sessionSupplier.get());
QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend());
for(QRecord qRecord : CollectionUtils.nonNullList(getBackendVariantFilteredRecords(process)))
QScheduleMetaData scheduleMetaData = process.getSchedule();
if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy()))
{
try
///////////////////////////////////////////////
// if no variants, or variant is serial mode //
///////////////////////////////////////////////
startProcess(process, null);
}
else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy()))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////
// if this a "parallel", which for example means we want to have a thread for each backend variant //
// running at the same time, get the variant records and schedule each separately //
/////////////////////////////////////////////////////////////////////////////////////////////////////
QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend());
for(QRecord qRecord : CollectionUtils.nonNullList(getBackendVariantFilteredRecords(process)))
{
startProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())));
}
catch(Exception e)
{
LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord));
try
{
startProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())));
}
catch(Exception e)
{
LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord));
}
}
}
else
{
LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided.");
}
}
else
{
LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided.");
}
}
}
finally
{
if(needToClearContext)
{
QContext.clear();
}
}
}
@ -210,8 +226,8 @@ public class ScheduleManager
// ask the PollingAutomationPerTableRunner how many threads of itself need setup //
// then start a scheduled executor foreach one //
///////////////////////////////////////////////////////////////////////////////////
List<PollingAutomationPerTableRunner.TableActions> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName());
for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions)
List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName());
for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
{
if(allowedToStart(tableAction.tableName()))
{

8
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java
View File

@ -574,7 +574,7 @@ public class ValueUtils
/*******************************************************************************
**
*******************************************************************************/
public static LocalTime getValueAsLocalTime(Serializable value)
public static LocalTime getValueAsLocalTime(Object value)
{
try
{
@ -615,7 +615,7 @@ public class ValueUtils
/*******************************************************************************
**
*******************************************************************************/
public static byte[] getValueAsByteArray(Serializable value)
public static byte[] getValueAsByteArray(Object value)
{
if(value == null)
{
@ -641,7 +641,7 @@ public class ValueUtils
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public static <T extends Serializable> T getValueAsType(Class<T> type, Serializable value)
public static <T extends Serializable> T getValueAsType(Class<T> type, Object value)
{
if(type.equals(Integer.class))
{
@ -687,7 +687,7 @@ public class ValueUtils
**
*******************************************************************************/
@SuppressWarnings("checkstyle:indentation")
public static Serializable getValueAsFieldType(QFieldType type, Serializable value)
public static Serializable getValueAsFieldType(QFieldType type, Object value)
{
return switch(type)
{