QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-20 22:18:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged dev into feature/CE-876-develop-missing-widget-types

Browse Source
This commit is contained in:
darin.kelkhoff
2024-02-20 20:12:55 -06:00
parent a6d84c66b7 73ac3448dd
commit bf11ea5185
35 changed files with 2822 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java
View File

@ -78,6 +78,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
private static Set<String> loggedUnauditableTableNames = new HashSet<>();
/*******************************************************************************
**
*******************************************************************************/
@ -210,6 +211,19 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
contextSuffix.append(" ").append(input.getAuditContext());
}
//////////////////////////////////////////////////////////////
// look for a context value place directly into the session //
//////////////////////////////////////////////////////////////
QSession qSession = QContext.getQSession();
if(qSession != null)
{
String sessionContext = qSession.getValue(AUDIT_CONTEXT_FIELD_NAME);
if(StringUtils.hasContent(sessionContext))
{
contextSuffix.append(" ").append(sessionContext);
}
}
/////////////////////////////////////////////////////////////////////////////////////
// note process label (and a possible context from the process's state) if present //
/////////////////////////////////////////////////////////////////////////////////////
@ -233,17 +247,20 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
///////////////////////////////////////////////////
// use api label & version if present in session //
///////////////////////////////////////////////////
QSession qSession = QContext.getQSession();
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
if(qSession != null)
{
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
{
apiLabel = "API";
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
{
apiLabel = "API";
}
contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
}
contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
}
return (contextSuffix.toString());
}

35
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.actions.automation;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
@ -55,6 +56,30 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
/*******************************************************************************
** Get instance by id
**
*******************************************************************************/
public static AutomationStatus getById(Integer id)
{
if(id == null)
{
return (null);
}
for(AutomationStatus value : AutomationStatus.values())
{
if(Objects.equals(value.id, id))
{
return (value);
}
}
return (null);
}
/*******************************************************************************
** Getter for id
**
@ -106,10 +131,10 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
public String getInsertOrUpdate()
{
return switch(this)
{
case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
case OK -> "";
};
{
case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
case OK -> "";
};
}
}

132
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java
View File

@ -22,30 +22,40 @@
package com.kingsrook.qqq.backend.core.actions.automation;
import java.io.Serializable;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
import org.apache.commons.lang.NotImplementedException;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -55,19 +65,37 @@ public class RecordAutomationStatusUpdater
{
private static final QLogger LOG = QLogger.getLogger(RecordAutomationStatusUpdater.class);
///////////////////////////////////////////////////////////////////////////////////////////////////////
// feature flag - by default, will be true - before setting records to PENDING_UPDATE_AUTOMATIONS, //
// we will fetch them (if we didn't take them in from the caller, which, UpdateAction does if its //
// backend supports it), to check their current automationStatus - and if they are currently PENDING //
// or RUNNING inserts or updates, we won't update them. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
private static boolean allowPreUpdateFetch = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.allowPreUpdateFetch", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_ALLOW_PRE_UPDATE_FETCH", true);
///////////////////////////////////////////////////////////////////////////////////////////////
// feature flag - by default, we'll memoize the check for triggers - but we can turn it off. //
///////////////////////////////////////////////////////////////////////////////////////////////
private static boolean memoizeCheckForTriggers = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.memoizeCheckForTriggers", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_MEMOIZE_CHECK_FOR_TRIGGERS", true);
private static Memoization<Key, Boolean> areThereTableTriggersForTableMemoization = new Memoization<Key, Boolean>().withTimeout(Duration.of(60, ChronoUnit.SECONDS));
/*******************************************************************************
** for a list of records from a table, set their automation status - based on
** how the table is configured.
*******************************************************************************/
public static boolean setAutomationStatusInRecords(QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus)
public static boolean setAutomationStatusInRecords(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction, List<QRecord> oldRecordList)
{
if(table == null || table.getAutomationDetails() == null || CollectionUtils.nullSafeIsEmpty(records))
{
return (false);
}
QTableAutomationDetails automationDetails = table.getAutomationDetails();
Set<Serializable> pkeysWeMayNotUpdate = new HashSet<>();
///////////////////////////////////////////////////////////////////////////////////////////////////
// In case an automation is running, and it updates records - don't let those records be marked //
// as PENDING_UPDATE_AUTOMATIONS... this is meant to avoid having a record's automation update //
@ -81,12 +109,60 @@ public class RecordAutomationStatusUpdater
for(StackTraceElement stackTraceElement : e.getStackTrace())
{
String className = stackTraceElement.getClassName();
if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test"))
if(className.contains(RecordAutomationStatusUpdater.class.getPackageName()) && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test") && !className.contains("Test$"))
{
LOG.debug("Avoiding re-setting automation status to PENDING_UPDATE while running an automation");
return (false);
}
}
///////////////////////////////////////////////////////////////////////////////
// if table uses field-in-table status tracking, then check the old records, //
// before we set them to pending-updates, to avoid losing other pending or //
// running status information. We will allow moving from OK or the 2 //
// failed statuses into pending-updates - which seems right. //
// This is added to fix cases where an update that comes in before insert //
// -automations have run, will cause the pending-insert status to be missed. //
///////////////////////////////////////////////////////////////////////////////
if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
{
try
{
if(CollectionUtils.nullSafeIsEmpty(oldRecordList))
{
///////////////////////////////////////////////////////////////////////////////////////////////
// if we didn't get the oldRecordList as input (though UpdateAction should usually pass it?) //
// then check feature-flag if we're allowed to do a lookup here & now. If so, then do. //
///////////////////////////////////////////////////////////////////////////////////////////////
if(allowPreUpdateFetch)
{
List<Serializable> pkeysToLookup = records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList();
oldRecordList = new QueryAction().execute(new QueryInput(table.getName())
.withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, pkeysToLookup)))
.withTransaction(transaction)
).getRecords();
}
}
for(QRecord freshRecord : CollectionUtils.nonNullList(oldRecordList))
{
Serializable recordStatus = freshRecord.getValue(automationDetails.getStatusTracking().getFieldName());
if(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
|| AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(recordStatus)
|| AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
|| AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId().equals(recordStatus))
{
Serializable primaryKey = freshRecord.getValue(table.getPrimaryKeyField());
LOG.debug("May not update automation status", logPair("table", table.getName()), logPair("id", primaryKey), logPair("currentStatus", recordStatus), logPair("requestedStatus", automationStatus.getId()));
pkeysWeMayNotUpdate.add(primaryKey);
}
}
}
catch(QException qe)
{
LOG.error("Error checking existing automation status before setting new automation status - more records will be updated than maybe should be...", qe);
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -98,19 +174,15 @@ public class RecordAutomationStatusUpdater
automationStatus = AutomationStatus.OK;
}
QTableAutomationDetails automationDetails = table.getAutomationDetails();
if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
{
for(QRecord record : records)
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// todo - seems like there's some case here, where if an order was in PENDING_INSERT, but then some other job updated the record, that we'd //
// lose that pending status, which would be a Bad Thing™... //
// problem is - we may not have the full record in here, so we can't necessarily check the record to see what status it's currently in... //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
// todo - another field - for the automation timestamp??
if(!pkeysWeMayNotUpdate.contains(record.getValue(table.getPrimaryKeyField())))
{
record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
// todo - another field - for the automation timestamp??
}
}
}
@ -188,11 +260,29 @@ public class RecordAutomationStatusUpdater
return (false);
}
if(memoizeCheckForTriggers)
{
///////////////////////////////////////////////////////////////////////////////////////
// as within the lookup method, error on the side of "yes, maybe there are triggers" //
///////////////////////////////////////////////////////////////////////////////////////
Optional<Boolean> result = areThereTableTriggersForTableMemoization.getResult(new Key(table, triggerEvent), key -> lookupIfThereAreTriggersForTable(table, triggerEvent));
return result.orElse(true);
}
else
{
return lookupIfThereAreTriggersForTable(table, triggerEvent);
}
}
/*******************************************************************************
**
*******************************************************************************/
private static Boolean lookupIfThereAreTriggersForTable(QTableMetaData table, TriggerEvent triggerEvent)
{
try
{
///////////////////
// todo - cache? //
///////////////////
CountInput countInput = new CountInput();
countInput.setTableName(TableTrigger.TABLE_NAME);
countInput.setFilter(new QQueryFilter(
@ -207,6 +297,7 @@ public class RecordAutomationStatusUpdater
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the count query failed, we're a bit safer to err on the side of "yeah, there might be automations" //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
LOG.warn("Error looking if there are triggers for table", e, logPair("tableName", table.getName()));
return (true);
}
}
@ -217,12 +308,12 @@ public class RecordAutomationStatusUpdater
** for a list of records, update their automation status and actually Update the
** backend as well.
*******************************************************************************/
public static void setAutomationStatusInRecordsAndUpdate(QInstance instance, QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus) throws QException
public static void setAutomationStatusInRecordsAndUpdate(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction) throws QException
{
QTableAutomationDetails automationDetails = table.getAutomationDetails();
if(automationDetails != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
{
boolean didSetStatusField = setAutomationStatusInRecords(session, table, records, automationStatus);
boolean didSetStatusField = setAutomationStatusInRecords(table, records, automationStatus, transaction, null);
if(didSetStatusField)
{
UpdateInput updateInput = new UpdateInput();
@ -237,6 +328,7 @@ public class RecordAutomationStatusUpdater
.withValue(table.getPrimaryKeyField(), r.getValue(table.getPrimaryKeyField()))
.withValue(automationDetails.getStatusTracking().getFieldName(), r.getValue(automationDetails.getStatusTracking().getFieldName()))).toList());
updateInput.setAreAllValuesBeingUpdatedTheSame(true);
updateInput.setTransaction(transaction);
updateInput.setOmitDmlAudit(true);
new UpdateAction().execute(updateInput);
@ -250,4 +342,8 @@ public class RecordAutomationStatusUpdater
}
}
private record Key(QTableMetaData table, TriggerEvent triggerEvent) {}
}

64
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java
View File

@ -28,6 +28,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
@ -60,6 +61,8 @@ import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
@ -252,12 +255,11 @@ public class PollingAutomationPerTableRunner implements Runnable
try
{
QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession();
processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), session, tableActions.status());
processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), tableActions.status());
}
catch(Exception e)
{
LOG.warn("Error running automations", e);
LOG.warn("Error running automations", e, logPair("tableName", tableActions.tableName()), logPair("status", tableActions.status()));
}
finally
{
@ -271,7 +273,7 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table.
*******************************************************************************/
public void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException
public void processTableInsertOrUpdate(QTableMetaData table, AutomationStatus automationStatus) throws QException
{
/////////////////////////////////////////////////////////////////////////
// get the actions to run against this table in this automation status //
@ -301,7 +303,9 @@ public class PollingAutomationPerTableRunner implements Runnable
AutomationStatusTrackingType statusTrackingType = automationDetails.getStatusTracking().getType();
if(AutomationStatusTrackingType.FIELD_IN_TABLE.equals(statusTrackingType))
{
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId()))));
QQueryFilter filter = new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId())));
addOrderByToQueryFilter(table, automationStatus, filter);
queryInput.setFilter(filter);
}
else
{
@ -322,7 +326,7 @@ public class PollingAutomationPerTableRunner implements Runnable
}, () ->
{
List<QRecord> records = recordPipe.consumeAvailableRecords();
applyActionsToRecords(session, table, records, actions, automationStatus);
applyActionsToRecords(table, records, actions, automationStatus);
return (records.size());
}
);
@ -330,6 +334,38 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
**
*******************************************************************************/
static void addOrderByToQueryFilter(QTableMetaData table, AutomationStatus automationStatus, QQueryFilter filter)
{
////////////////////////////////////////////////////////////////////////////////////
// look for a field in the table with either create-date or modify-date behavior, //
// based on if doing insert or update automations //
////////////////////////////////////////////////////////////////////////////////////
DynamicDefaultValueBehavior dynamicDefaultValueBehavior = automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS) ? DynamicDefaultValueBehavior.CREATE_DATE : DynamicDefaultValueBehavior.MODIFY_DATE;
Optional<QFieldMetaData> field = table.getFields().values().stream()
.filter(f -> dynamicDefaultValueBehavior.equals(f.getBehaviorOrDefault(QContext.getQInstance(), DynamicDefaultValueBehavior.class)))
.findFirst();
if(field.isPresent())
{
//////////////////////////////////////////////////////////////////////
// if a create/modify date field was found, order by it (ascending) //
//////////////////////////////////////////////////////////////////////
filter.addOrderBy(new QFilterOrderBy(field.get().getName()));
}
else
{
////////////////////////////////////
// else, order by the primary key //
////////////////////////////////////
filter.addOrderBy(new QFilterOrderBy(table.getPrimaryKeyField()));
}
}
/*******************************************************************************
** get the actions to run against a table in an automation status. both from
** metaData and tableTriggers/data.
@ -430,7 +466,7 @@ public class PollingAutomationPerTableRunner implements Runnable
** table's actions against them - IF they are found to match the action's filter
** (assuming it has one - if it doesn't, then all records match).
*******************************************************************************/
private void applyActionsToRecords(QSession session, QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
private void applyActionsToRecords(QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
{
if(CollectionUtils.nullSafeIsEmpty(records))
{
@ -440,7 +476,7 @@ public class PollingAutomationPerTableRunner implements Runnable
///////////////////////////////////////////////////
// mark the records as RUNNING their automations //
///////////////////////////////////////////////////
RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToRunningStatusMap.get(automationStatus));
RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, pendingToRunningStatusMap.get(automationStatus), null);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
// foreach action - run it against the records (but only if they match the action's filter, if there is one) //
@ -458,13 +494,15 @@ public class PollingAutomationPerTableRunner implements Runnable
////////////////////////////////////////
// update status on all these records //
////////////////////////////////////////
if(anyActionsFailed)
AutomationStatus statusToUpdateTo = anyActionsFailed ? pendingToFailedStatusMap.get(automationStatus) : AutomationStatus.OK;
try
{
RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus));
RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, statusToUpdateTo, null);
}
else
catch(Exception e)
{
RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK);
LOG.warn("Error updating automationStatus after running automations", logPair("tableName", table), logPair("count", records.size()), logPair("status", statusToUpdateTo));
throw (e);
}
}
@ -494,7 +532,7 @@ public class PollingAutomationPerTableRunner implements Runnable
}
catch(Exception e)
{
LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
LOG.warn("Caught exception processing automations", e, logPair("tableName", table), logPair("action", action.getName()));
return (true);
}
}

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java
View File

@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.QQueryFilterDeduper;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -176,11 +177,13 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
{
return (totalString);
}
filter = QQueryFilterDeduper.dedupeFilter(filter);
return ("<a href='" + tablePath + "?filter=" + JsonUtils.toJson(filter) + "'>" + totalString + "</a>");
}
/*******************************************************************************
**
*******************************************************************************/
@ -192,6 +195,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
return;
}
filter = QQueryFilterDeduper.dedupeFilter(filter);
urls.add(tablePath + "?filter=" + JsonUtils.toJson(filter));
}
@ -208,6 +212,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
return (null);
}
filter = QQueryFilterDeduper.dedupeFilter(filter);
return (tablePath + "?filter=" + JsonUtils.toJson(filter));
}
@ -224,6 +229,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
return (null);
}
filter = QQueryFilterDeduper.dedupeFilter(filter);
return (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
}
@ -326,6 +332,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
}
String tablePath = QContext.getQInstance().getTablePath(tableName);
filter = QQueryFilterDeduper.dedupeFilter(filter);
return (tablePath + "/" + processName + "?recordsParam=filterJSON&filterJSON=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8));
}

55
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java
View File

@ -30,6 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
@ -47,12 +48,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.scripts.Script;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
@ -65,6 +68,8 @@ public class RunAdHocRecordScriptAction
private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptRevisionId = new HashMap<>();
private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptId = new HashMap<>();
private static Memoization<Integer, Script> scriptMemoizationById = new Memoization<>();
/*******************************************************************************
@ -85,6 +90,12 @@ public class RunAdHocRecordScriptAction
throw (new QException("Script revision was not found."));
}
Optional<Script> script = getScript(scriptRevision);
QContext.getQSession().setValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, script.isPresent()
? "via Script \"%s\"".formatted(script.get().getName())
: "via Script id " + scriptRevision.getScriptId());
////////////////////////////
// figure out the records //
////////////////////////////
@ -124,6 +135,10 @@ public class RunAdHocRecordScriptAction
{
output.setException(Optional.of(e));
}
finally
{
QContext.getQSession().removeValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME);
}
}
@ -217,4 +232,44 @@ public class RunAdHocRecordScriptAction
throw (new QException("Code reference did not contain a scriptRevision, scriptRevisionId, or scriptId"));
}
/*******************************************************************************
**
*******************************************************************************/
private Optional<Script> getScript(ScriptRevision scriptRevision)
{
if(scriptRevision == null || scriptRevision.getScriptId() == null)
{
return (Optional.empty());
}
try
{
return scriptMemoizationById.getResult(scriptRevision.getScriptId(), scriptId ->
{
try
{
QRecord scriptRecord = new GetAction().executeForRecord(new GetInput(Script.TABLE_NAME).withPrimaryKey(scriptRevision.getScriptId()));
if(scriptRecord != null)
{
Script script = new Script(scriptRecord);
scriptMemoizationById.storeResult(scriptRevision.getScriptId(), script);
return (script);
}
}
catch(Exception e)
{
LOG.info("");
}
return (null);
});
}
catch(Exception e)
{
return (Optional.empty());
}
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -444,7 +444,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
*******************************************************************************/
private void setAutomationStatusField(InsertInput insertInput)
{
RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getSession(), insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS);
RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, insertInput.getTransaction(), null);
}

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java
View File

@ -113,7 +113,6 @@ public class UpdateAction
public UpdateOutput execute(UpdateInput updateInput) throws QException
{
ActionHelper.validateSession(updateInput);
setAutomationStatusField(updateInput);
QTableMetaData table = updateInput.getTable();
@ -130,6 +129,16 @@ public class UpdateAction
////////////////////////////////////////////////////////////////////////////////
Optional<List<QRecord>> oldRecordList = fetchOldRecords(updateInput, updateInterface);
///////////////////////////////////////////////////////////////////////////////////////
// allow caller to specify that we don't want to trigger automations. this isn't //
// isn't expected to be used much - by design, only for the process that is meant to //
// heal automation status, so that it can force us into status=Pending-inserts //
///////////////////////////////////////////////////////////////////////////////////////
if(!updateInput.getOmitTriggeringAutomations())
{
setAutomationStatusField(updateInput, oldRecordList);
}
performValidations(updateInput, oldRecordList, false);
////////////////////////////////////
@ -561,9 +570,9 @@ public class UpdateAction
/*******************************************************************************
** If the table being updated uses an automation-status field, populate it now.
*******************************************************************************/
private void setAutomationStatusField(UpdateInput updateInput)
private void setAutomationStatusField(UpdateInput updateInput, Optional<List<QRecord>> oldRecordList)
{
RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getSession(), updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS);
RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, updateInput.getTransaction(), oldRecordList.orElse(null));
}
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java
View File

@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.serialization.QFilterCriteriaDeserializer;
@ -346,4 +347,37 @@ public class QFilterCriteria implements Serializable, Cloneable
return (rs.toString());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public boolean equals(Object o)
{
if(this == o)
{
return true;
}
if(o == null || getClass() != o.getClass())
{
return false;
}
QFilterCriteria that = (QFilterCriteria) o;
return Objects.equals(fieldName, that.fieldName) && operator == that.operator && Objects.equals(values, that.values) && Objects.equals(otherFieldName, that.otherFieldName);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public int hashCode()
{
return Objects.hash(fieldName, operator, values, otherFieldName);
}
}

45
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java
View File

@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -323,6 +324,18 @@ public class QQueryFilter implements Serializable, Cloneable
/*******************************************************************************
** Fluent setter for adding a single subFilter
**
*******************************************************************************/
public QQueryFilter withSubFilter(QQueryFilter subFilter)
{
addSubFilter(subFilter);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@ -467,4 +480,36 @@ public class QQueryFilter implements Serializable, Cloneable
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public boolean equals(Object o)
{
if(this == o)
{
return true;
}
if(o == null || getClass() != o.getClass())
{
return false;
}
QQueryFilter that = (QQueryFilter) o;
return Objects.equals(criteria, that.criteria) && Objects.equals(orderBys, that.orderBys) && booleanOperator == that.booleanOperator && Objects.equals(subFilters, that.subFilters) && Objects.equals(skip, that.skip) && Objects.equals(limit, that.limit);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public int hashCode()
{
return Objects.hash(criteria, orderBys, booleanOperator, subFilters, skip, limit);
}
}

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java
View File

@ -51,6 +51,7 @@ public class UpdateInput extends AbstractTableActionInput
////////////////////////////////////////////////////////////////////////////////////////////
private Boolean areAllValuesBeingUpdatedTheSame = null;
private boolean omitTriggeringAutomations = false;
private boolean omitDmlAudit = false;
private String auditContext = null;
@ -321,4 +322,35 @@ public class UpdateInput extends AbstractTableActionInput
return (this);
}
/*******************************************************************************
** Getter for omitTriggeringAutomations
*******************************************************************************/
public boolean getOmitTriggeringAutomations()
{
return (this.omitTriggeringAutomations);
}
/*******************************************************************************
** Setter for omitTriggeringAutomations
*******************************************************************************/
public void setOmitTriggeringAutomations(boolean omitTriggeringAutomations)
{
this.omitTriggeringAutomations = omitTriggeringAutomations;
}
/*******************************************************************************
** Fluent setter for omitTriggeringAutomations
*******************************************************************************/
public UpdateInput withOmitTriggeringAutomations(boolean omitTriggeringAutomations)
{
this.omitTriggeringAutomations = omitTriggeringAutomations;
return (this);
}
}

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java
View File

@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -167,6 +168,11 @@ public class QRecord implements Serializable
ArrayList<?> cloneList = new ArrayList<>(arrayList);
clone.put(entry.getKey(), (V) cloneList);
}
else if(entry.getValue() instanceof LinkedList<?> linkedList)
{
LinkedList<?> cloneList = new LinkedList<>(linkedList);
clone.put(entry.getKey(), (V) cloneList);
}
else if(entry.getValue() instanceof LinkedHashMap<?, ?> linkedHashMap)
{
LinkedHashMap<?, ?> cloneMap = new LinkedHashMap<>(linkedHashMap);

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/HtmlWrapper.java
View File

@ -51,6 +51,7 @@ public class HtmlWrapper implements Serializable
public static final String STYLE_INDENT_2 = "padding-left: 2rem; ";
public static final String STYLE_FLOAT_RIGHT = "float: right; ";
public static final String STYLE_RED = "color: red; ";
public static final String STYLE_YELLOW = "color: #bfb743; ";

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java
View File

@ -142,6 +142,19 @@ public class QSession implements Serializable
/*******************************************************************************
**
*******************************************************************************/
public void removeValue(String key)
{
if(values != null)
{
values.remove(key);
}
}
/*******************************************************************************
**
*******************************************************************************/

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryUpdateAction.java
View File

@ -22,13 +22,10 @@
package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
@ -45,17 +42,6 @@ public class MemoryUpdateAction extends AbstractMemoryAction implements UpdateIn
{
try
{
QTableMetaData table = updateInput.getTable();
Instant now = Instant.now();
for(QRecord record : updateInput.getRecords())
{
///////////////////////////////////////////
// todo .. better (not hard-coded names) //
///////////////////////////////////////////
setValueIfTableHasField(record, table, "modifyDate", now, false);
}
UpdateOutput updateOutput = new UpdateOutput();
updateOutput.setRecords(MemoryRecordStore.getInstance().update(updateInput, true));
return (updateOutput);

298
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java Normal file
View File

@ -0,0 +1,298 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.automation;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.HtmlWrapper;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MultiLevelMapHelper;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Process to find records with a bad automation status, and repair them.
**
** Bad status are defined as:
** - failed insert or updates.
** - running insert or updates for more than X minutes (see input field value).
**
** Repair in this case means resetting their status to the corresponding (e.g.,
** insert/update) pending status.
**
*******************************************************************************/
public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, MetaDataProducerInterface<QProcessMetaData>
{
public static final String NAME = "HealBadRecordAutomationStatusesProcess";
private static final QLogger LOG = QLogger.getLogger(HealBadRecordAutomationStatusesProcessStep.class);
private static final Map<Integer, Integer> statusUpdateMap = Map.of(
AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(),
AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(),
AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId(),
AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId()
);
/*******************************************************************************
**
*******************************************************************************/
@Override
public QProcessMetaData produce(QInstance qInstance) throws QException
{
QProcessMetaData processMetaData = new QProcessMetaData()
.withName(NAME)
.withStepList(List.of(
new QFrontendStepMetaData()
.withName("input")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
.withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
.withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)),
new QBackendStepMetaData()
.withName("run")
.withCode(new QCodeReference(getClass())),
new QFrontendStepMetaData()
.withName("output")
.withComponent(new NoCodeWidgetFrontendComponentMetaData()
.withOutput(new WidgetHtmlLine()
.withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
.withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
.withVelocityTemplate("<b>Warning:</b>"))
.withOutput(new WidgetHtmlLine()
.withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
.withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1))
.withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
.withVelocityTemplate("""
<ul>
#foreach($string in $warnings)
<li>$string</li>
#end
</ul>
""")))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
.withViewField(new QFieldMetaData("totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
.withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
.withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
.withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */)
));
return (processMetaData);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
int recordsUpdated = 0;
////////////////////////////////////////////////////////////////////////
// if a table name is given, validate it, and run for just that table //
////////////////////////////////////////////////////////////////////////
String tableName = runBackendStepInput.getValueString("tableName");
ArrayList<String> warnings = new ArrayList<>();
if(StringUtils.hasContent(tableName))
{
if(!QContext.getQInstance().getTables().containsKey(tableName))
{
throw (new QException("Unrecognized table name: " + tableName));
}
recordsUpdated += processTable(tableName, runBackendStepInput, runBackendStepOutput, warnings);
}
else
{
//////////////////////////////////////////////////////////////////////////
// else, try to run for all tables that have an automation status field //
//////////////////////////////////////////////////////////////////////////
for(QTableMetaData table : QContext.getQInstance().getTables().values())
{
recordsUpdated += processTable(table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
}
}
runBackendStepOutput.addValue("totalRecordsUpdated", recordsUpdated);
runBackendStepOutput.addValue("warnings", warnings);
runBackendStepOutput.addValue("warningCount", warnings.size());
if(CollectionUtils.nullSafeIsEmpty(runBackendStepOutput.getRecords()))
{
runBackendStepOutput.addRecord(new QRecord()
.withValue("tableName", "--")
.withValue("badStatus", "--")
.withValue("count", "0"));
}
}
/*******************************************************************************
**
*******************************************************************************/
private int processTable(String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
{
try
{
Integer minutesOldLimit = Objects.requireNonNullElse(runBackendStepInput.getValueInteger("minutesOldLimit"), 60);
QTableMetaData table = QContext.getQInstance().getTable(tableName);
//////////////////////////////////////////////////////////////////////////
// only process tables w/ automation details w/ a status tracking field //
//////////////////////////////////////////////////////////////////////////
if(table != null && table.getAutomationDetails() != null && table.getAutomationDetails().getStatusTracking() != null && StringUtils.hasContent(table.getAutomationDetails().getStatusTracking().getFieldName()))
{
String automationStatusFieldName = table.getAutomationDetails().getStatusTracking().getFieldName();
/////////////////////////////////////////////
// find the modify-date field on the table //
/////////////////////////////////////////////
String modifyDateFieldName = null;
for(QFieldMetaData field : table.getFields().values())
{
if(DynamicDefaultValueBehavior.MODIFY_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
{
modifyDateFieldName = field.getName();
break;
}
}
if(modifyDateFieldName == null)
{
warnings.add("Could not find a Modify Date field on table: " + tableName);
LOG.info("Couldn't find a MODIFY_DATE field on table", logPair("tableName", tableName));
return 0;
}
////////////////////////////////////////////////////////////////////////
// query for records either FAILED, or RUNNING w/ modify date too old //
////////////////////////////////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
.withSubFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())))
.withSubFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
.withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))))
);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// foreach record found, add it to list of records to be updated - mapping status to appropriate pending status //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsToUpdate = new ArrayList<>();
Map<String, Integer> countByStatus = new HashMap<>();
for(QRecord record : queryOutput.getRecords())
{
Integer badAutomationStatusId = record.getValueInteger(automationStatusFieldName);
Integer updateStatus = statusUpdateMap.get(badAutomationStatusId);
if(updateStatus != null)
{
AutomationStatus badStatus = AutomationStatus.getById(badAutomationStatusId);
if(badStatus != null)
{
MultiLevelMapHelper.getOrPutAndIncrement(countByStatus, badStatus.getLabel());
}
recordsToUpdate.add(new QRecord()
.withValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField()))
.withValue(automationStatusFieldName, updateStatus));
}
}
if(!recordsToUpdate.isEmpty())
{
LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size()));
new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true));
}
for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
{
runBackendStepOutput.addRecord(new QRecord()
.withValue("tableName", tableName)
.withValue("badStatus", entry.getKey())
.withValue("count", entry.getValue()));
}
return (recordsToUpdate.size());
}
}
catch(Exception e)
{
warnings.add("Error processing table: " + tableName + ": " + ExceptionUtils.getTopAndBottomMessages(e));
LOG.warn("Error processing table for bad automation statuses", e, logPair("tableName, name"));
}
return 0;
}
}

149
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java Normal file
View File

@ -0,0 +1,149 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.automation;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Process to manually run table automations, for a table.
**
** Useful, maybe, for an e2e test. Or, if you don't want jobs to be running,
** but want to run automations by-hand, for some reason.
**
** In the future, this class could take a param to only do inserts or updates.
**
** Also, right now, only records that are Pending automations will be run -
** again, that could be changed, presumably (take a list of records, always run, etc...)
*******************************************************************************/
public class RunTableAutomationsProcessStep implements BackendStep, MetaDataProducerInterface<QProcessMetaData>
{
public static final String NAME = "RunTableAutomationsProcess";
/*******************************************************************************
**
*******************************************************************************/
@Override
public QProcessMetaData produce(QInstance qInstance) throws QException
{
QProcessMetaData processMetaData = new QProcessMetaData()
.withName(NAME)
.withStepList(List.of(
new QFrontendStepMetaData()
.withName("input")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
.withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
.withFormField(new QFieldMetaData("automationProviderName", QFieldType.STRING)),
new QBackendStepMetaData()
.withName("run")
.withCode(new QCodeReference(getClass())),
new QFrontendStepMetaData()
.withName("output")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
.withViewField(new QFieldMetaData("ok", QFieldType.STRING))
));
return (processMetaData);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
QInstance qInstance = QContext.getQInstance();
////////////////////////////////////////////////////////////////////
// get tableName param (since this process is not table-specific) //
////////////////////////////////////////////////////////////////////
String tableName = runBackendStepInput.getValueString("tableName");
if(!StringUtils.hasContent(tableName))
{
throw (new QException("Missing required input value: tableName"));
}
if(!QContext.getQInstance().getTables().containsKey(tableName))
{
throw (new QException("Unrecognized table name: " + tableName));
}
////////////////////////////////////////////////////////////////////////////////////////////////
// get the automation provider name to use - either as the only-one-in-instance, or via param //
////////////////////////////////////////////////////////////////////////////////////////////////
String automationProviderName = runBackendStepInput.getValueString("automationProviderName");
if(!StringUtils.hasContent(automationProviderName))
{
Map<String, QAutomationProviderMetaData> automationProviders = CollectionUtils.nonNullMap(qInstance.getAutomationProviders());
if(automationProviders.size() == 1)
{
automationProviderName = automationProviders.keySet().iterator().next();
}
else
{
throw (new QException("Missing required input value: automationProviderName (and there is not exactly 1 in the active instance)"));
}
}
/////////////////////////////////////////////
// run automations for the requested table //
/////////////////////////////////////////////
List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProviderName);
for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
{
if(tableName.equals(tableAction.tableName()))
{
PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunner(qInstance, automationProviderName, () -> QContext.getQSession(), tableAction);
pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), tableAction.status());
}
}
runBackendStepOutput.addValue("ok", "true");
}
}

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java
View File

@ -26,7 +26,6 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
@ -133,11 +132,6 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
throw (new QException("Could not find script by id: " + scriptId));
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set an "audit context" - so any DML executed during the script will include the note of what script was running. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"" + script.getValue("name") + "\"");
String tableName = script.getValueString("tableName");
RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();

19
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ExceptionUtils.java
View File

@ -164,4 +164,23 @@ public class ExceptionUtils
return (StringUtils.join("; ", messages));
}
/*******************************************************************************
** Get the messages from the top & bottoms (root) of an exception.
**
** If there's no root, just return the top (e.g., parameter)'s message.
** If they are both found, put ": " between them.
*******************************************************************************/
public static String getTopAndBottomMessages(Exception e)
{
String rs = e.getMessage();
Throwable rootException = getRootException(e);
if(rootException != e)
{
rs += ": " + rootException.getMessage();
}
return (rs);
}
}

363
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduper.java Normal file
View File

@ -0,0 +1,363 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.utils;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS;
import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN;
import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_EQUALS;
import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_IN;
/*******************************************************************************
** Class to help deduplicate redundant criteria in filters.
**
** Original use-case is for making more clean url links out of filters.
**
** Does not (at this time) look into sub-filters at all, or support any "OR"
** filters other than the most basic (a=1 OR a=1).
**
** Also, other than for completely redundant criteria (e.g., a>1 and a>1) only
* works on a limited subset of criteria operators (EQUALS, NOT_EQUALS, IN, and NOT_IN)
*******************************************************************************/
public class QQueryFilterDeduper
{
private static final QLogger LOG = QLogger.getLogger(QQueryFilterDeduper.class);
/*******************************************************************************
**
*******************************************************************************/
public static QQueryFilter dedupeFilter(QQueryFilter filter)
{
if(filter == null)
{
return (null);
}
try
{
/////////////////////////////////////////////////////////////////
// track (just for logging) if we failed or if we did any good //
/////////////////////////////////////////////////////////////////
List<String> log = new ArrayList<>();
boolean fail = false;
boolean didAnyGood = false;
//////////////////////////////////////////////////////////////////////////////////////////
// always create a clone to be returned. this is especially useful because, //
// the clone's lists will be ArrayLists, which are mutable - since some of the deduping //
// involves manipulating value lists. //
//////////////////////////////////////////////////////////////////////////////////////////
QQueryFilter rs = filter.clone();
////////////////////////////////////////////////////////////////////////////////////
// general strategy is: //
// iterate over criteria, possibly removing the one the iterator is pointing at, //
// if we are able to somehow merge it into other criteria we've already seen. //
// the others-we've-seen will be tracked in the criteriaByFieldName listing hash. //
////////////////////////////////////////////////////////////////////////////////////
ListingHash<String, QFilterCriteria> criteriaByFieldName = new ListingHash<>();
Iterator<QFilterCriteria> iterator = rs.getCriteria().iterator();
while(iterator.hasNext())
{
QFilterCriteria criteria = iterator.next();
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first thing to check is, have we seen any other criteria for this field - if so - try to do some de-duping. //
// note that, any time we do a remove, we'll need to do a continue - to avoid adding the now-removed criteria //
// to the listing hash //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(criteriaByFieldName.containsKey(criteria.getFieldName()))
{
List<QFilterCriteria> others = criteriaByFieldName.get(criteria.getFieldName());
QFilterCriteria other = others.get(0);
if(others.size() == 1 && other.equals(criteria))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we've only see 1 other criteria for this field so far, and this one is an exact match, then remove this one. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
log.add(String.format("Remove duplicate criteria [%s]", criteria));
iterator.remove();
didAnyGood = true;
continue;
}
else
{
/////////////////////////////////////////////////////////////////////////////////////////
// else - if there's still just 1 other, and it's an AND query - then apply some basic //
// logic-merging operations, based on the pair of criteria operators //
/////////////////////////////////////////////////////////////////////////////////////////
if(others.size() == 1 && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
{
if((NOT_EQUALS.equals(other.getOperator()) || NOT_IN.equals(other.getOperator())) && EQUALS.equals(criteria.getOperator()))
{
///////////////////////////////////////////////////////////////////////////
// if we previously saw a not-equals or not-in, and now we see an equals //
// and the value from the EQUALS isn't in the not-in list //
// then replace the not-equals with the equals //
// then just discard this equals //
///////////////////////////////////////////////////////////////////////////
if(other.getValues().contains(criteria.getValues().get(0)))
{
log.add("Contradicting NOT_EQUALS/NOT_IN and EQUALS");
fail = true;
}
else
{
other.setOperator(criteria.getOperator());
other.setValues(criteria.getValues());
iterator.remove();
didAnyGood = true;
log.add("Replace a not-equals or not-in superseded by an equals");
continue;
}
}
else if(EQUALS.equals(other.getOperator()) && (NOT_EQUALS.equals(criteria.getOperator()) || NOT_IN.equals(criteria.getOperator())))
{
/////////////////////////////////////////////////////////////////////////////
// if we previously saw an equals, and now we see a not-equals or a not-in //
// and the value from the EQUALS isn't in the not-in list //
// then just discard this not-equals //
/////////////////////////////////////////////////////////////////////////////
if(criteria.getValues().contains(other.getValues().get(0)))
{
log.add("Contradicting NOT_EQUALS/NOT_IN and EQUALS");
fail = true;
}
else
{
iterator.remove();
didAnyGood = true;
log.add("Remove a redundant not-equals");
continue;
}
}
else if(NOT_EQUALS.equals(other.getOperator()) && IN.equals(criteria.getOperator()))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////
// if we previously saw a not-equals, and now we see an IN //
// then replace the not-equals with the IN (making sure the not-equals value isn't in the in-list) //
// then just discard this equals //
/////////////////////////////////////////////////////////////////////////////////////////////////////
Serializable notEqualsValue = other.getValues().get(0);
List<Serializable> inValues = new ArrayList<>(criteria.getValues());
inValues.remove(notEqualsValue);
if(inValues.isEmpty())
{
///////////////////////////////////////////////////////////////////////////////////
// if the only in-value was the not-equal value, then... i don't know, don't try //
///////////////////////////////////////////////////////////////////////////////////
log.add("Contradicting IN and NOT_EQUAL");
fail = true;
}
else
{
//////////////////////////////////////////////////////////////////
// else, we can proceed by replacing the not-equals with the in //
//////////////////////////////////////////////////////////////////
other.setOperator(criteria.getOperator());
other.setValues(criteria.getValues());
iterator.remove();
didAnyGood = true;
log.add("Replace superseded not-equals (removing its value from in-list)");
continue;
}
}
else if(IN.equals(other.getOperator()) && NOT_EQUALS.equals(criteria.getOperator()))
{
//////////////////////////////////////////////////////////////////
// if we previously saw an in, and now we see a not-equals //
// discard the not-equals (removing its value from the in-list) //
// then just discard this not-equals //
//////////////////////////////////////////////////////////////////
Serializable notEqualsValue = criteria.getValues().get(0);
List<Serializable> inValues = new ArrayList<>(other.getValues());
inValues.remove(notEqualsValue);
if(inValues.isEmpty())
{
///////////////////////////////////////////////////////////////////////////////////
// if the only in-value was the not-equal value, then... i don't know, don't try //
///////////////////////////////////////////////////////////////////////////////////
log.add("Contradicting IN and NOT_EQUAL");
fail = true;
}
else
{
//////////////////////////////////////////////////////////////////
// else, we can proceed by replacing the not-equals with the in //
//////////////////////////////////////////////////////////////////
iterator.remove();
didAnyGood = true;
log.add("Remove redundant not-equals (removing its value from in-list)");
continue;
}
}
else if(NOT_EQUALS.equals(other.getOperator()) && NOT_IN.equals(criteria.getOperator()))
{
/////////////////////////////////////////////////////////////////////////////////////////
// if we previously saw a not-equals, and now we see a not-in //
// we can change the not-equals to the not-in, and make sure it's value is in the list //
// then just discard this not-in //
/////////////////////////////////////////////////////////////////////////////////////////
Serializable originalNotEqualsValue = other.getValues().get(0);
other.setOperator(criteria.getOperator());
other.setValues(criteria.getValues());
if(!other.getValues().contains(originalNotEqualsValue))
{
other.getValues().add(originalNotEqualsValue);
}
iterator.remove();
didAnyGood = true;
log.add("Replace superseded not-equals with not-in");
continue;
}
else if(NOT_IN.equals(other.getOperator()) && NOT_EQUALS.equals(criteria.getOperator()))
{
////////////////////////////////////////////////////////////////////////////////////////
// if we previously saw a not-in, and now we see a not-equals //
// we can discard this not-equals, and just make sure its value is in the not-in list //
////////////////////////////////////////////////////////////////////////////////////////
Serializable originalNotEqualsValue = criteria.getValues().get(0);
if(!other.getValues().contains(originalNotEqualsValue))
{
other.getValues().add(originalNotEqualsValue);
}
iterator.remove();
didAnyGood = true;
log.add("Remove not-equals, absorbing into not-in");
continue;
}
else if(NOT_IN.equals(other.getOperator()) && NOT_IN.equals(criteria.getOperator()))
{
////////////////////////////////////////////////////////////////
// for multiple not-ins, just merge their values (as a union) //
////////////////////////////////////////////////////////////////
for(Serializable value : criteria.getValues())
{
if(!other.getValues().contains(value))
{
other.getValues().add(value);
}
}
iterator.remove();
didAnyGood = true;
log.add("Merging not-ins");
continue;
}
else if(IN.equals(other.getOperator()) && IN.equals(criteria.getOperator()))
{
////////////////////////////////////////////////////////////////////////
// for multiple not-ins, just merge their values (as an intersection) //
////////////////////////////////////////////////////////////////////////
Set<Serializable> otherValues = new HashSet<>(other.getValues());
Set<Serializable> criteriaValues = new HashSet<>(criteria.getValues());
otherValues.retainAll(criteriaValues);
if(otherValues.isEmpty())
{
log.add("Contradicting IN lists (no values)");
fail = true;
}
else
{
other.setValues(new ArrayList<>(otherValues));
iterator.remove();
didAnyGood = true;
log.add("Merging not-ins");
continue;
}
}
else if(NOT_EQUALS.equals(other.getOperator()) && NOT_EQUALS.equals(criteria.getOperator()))
{
/////////////////////////////////////////////////////////////////////////////////////
// if we have 2 not-equals, we can merge them in a not-in //
// we can assume their values are different, else they'd have been equals up above //
/////////////////////////////////////////////////////////////////////////////////////
other.setOperator(NOT_IN);
other.setValues(new ArrayList<>(List.of(other.getValues().get(0), criteria.getValues().get(0))));
iterator.remove();
didAnyGood = true;
log.add("Merge two not-equals as not-in");
continue;
}
else
{
log.add("Fail because unhandled operator pair");
fail = true;
}
}
else
{
log.add("Fail because > 1 other or operator: OR");
fail = true;
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we reach here (e.g., no continue), then assuming we didn't remove the criteria, add it to the listing hash. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
criteriaByFieldName.add(criteria.getFieldName(), criteria);
}
///////////////////////////
// log based on booleans //
///////////////////////////
if(fail && didAnyGood)
{
LOG.info("Partially unsuccessful dedupe of filter", logPair("original", filter), logPair("deduped", rs), logPair("log", log));
}
else if(fail)
{
LOG.info("Unsuccessful dedupe of filter", logPair("filter", filter), logPair("log", log));
}
else if(didAnyGood)
{
LOG.debug("Successful dedupe of filter", logPair("original", filter), logPair("deduped", rs), logPair("log", log));
}
else
{
LOG.debug("No duplicates in filter, so nothing to dedupe", logPair("original", filter));
}
return rs;
}
catch(Exception e)
{
LOG.warn("Error de-duping filter", e, logPair("filter", filter));
return (filter.clone());
}
}
}

104
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java
View File

@ -30,6 +30,7 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
/*******************************************************************************
@ -58,8 +59,16 @@ public class Memoization<K, V>
/*******************************************************************************
** Get the memoized Value for a given input Key.
**
** But note, this looks the same to the caller, whether the key just wasn't in
** the internal map (e.g., had never been looked up), or if it was previously looked
** up, and that returned null. In either case, the optional will be empty.
**
** See getMemoizedResult for where we can tell the difference (and we would
** generally want to call that.
*******************************************************************************/
@Deprecated
public Optional<V> getResult(K key)
{
MemoizedResult<V> result = map.get(key);
@ -77,7 +86,57 @@ public class Memoization<K, V>
/*******************************************************************************
** Get the memoized Value for a given input Key - computing it if it wasn't previously
** memoized (or expired).
**
** In here, if the optional is empty, it means the value is null (whether that
** came form memoization, or from the lookupFunction, you don't care - the answer
** is null).
*******************************************************************************/
public Optional<V> getResult(K key, UnsafeFunction<K, V, ?> lookupFunction)
{
MemoizedResult<V> result = map.get(key);
if(result != null)
{
if(result.getTime().isAfter(Instant.now().minus(timeout)))
{
//////////////////////////////////////////////////////////////////////////////
// ok, we have a memoized value, and it's not expired, so we can return it. //
// of course, it might be a memoized null, so we use .ofNullable. //
//////////////////////////////////////////////////////////////////////////////
return (Optional.ofNullable(result.getResult()));
}
}
/////////////////////////////////////////////////////////////////////////////////////////////
// ok - either we never memoized this key, or it's expired, so, apply the lookup function, //
// store the result, and then return the value (in an Optional.ofNullable) //
/////////////////////////////////////////////////////////////////////////////////////////////
try
{
V value = lookupFunction.apply(key);
storeResult(key, value);
return (Optional.ofNullable(value));
}
catch(Exception e)
{
LOG.warn("Uncaught Exception while executing a Memoization lookupFunction (to avoid this log, add a catch in the lookupFunction)", e);
storeResult(key, null);
return (Optional.empty());
}
}
/*******************************************************************************
** Get a memoized result, optionally containing a Value, for a given input Key.
**
** In this method (contrasted with getResult), if the returned Optional is empty,
** it means that we haven't ever looked up or memoized the key (or it's expired).
**
** If the returned Optional is not empty, then it means we've memoized something
** (and it's not expired) - so if the Value from the MemoizedResult is null,
** then null is the proper memoized value.
*******************************************************************************/
public Optional<MemoizedResult<V>> getMemoizedResult(K key)
{
@ -86,7 +145,7 @@ public class Memoization<K, V>
{
if(result.getTime().isAfter(Instant.now().minus(timeout)))
{
return (Optional.ofNullable(result));
return (Optional.of(result));
}
}
@ -181,4 +240,47 @@ public class Memoization<K, V>
{
return map;
}
/*******************************************************************************
** Getter for timeout
*******************************************************************************/
public Duration getTimeout()
{
return (this.timeout);
}
/*******************************************************************************
** Fluent setter for timeout
*******************************************************************************/
public Memoization<K, V> withTimeout(Duration timeout)
{
this.timeout = timeout;
return (this);
}
/*******************************************************************************
** Getter for maxSize
*******************************************************************************/
public Integer getMaxSize()
{
return (this.maxSize);
}
/*******************************************************************************
** Fluent setter for maxSize
*******************************************************************************/
public Memoization<K, V> withMaxSize(Integer maxSize)
{
this.maxSize = maxSize;
return (this);
}
}