QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged dev into feature/join-enhancements

Browse Source
This commit is contained in:
darin.kelkhoff
2023-09-27 09:16:08 -05:00
parent 831ac3bc07 71302eefdf
commit 623f3aca93
37 changed files with 2069 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java
View File

@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@ -49,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -108,12 +110,26 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
/*******************************************************************************
** Simple overload that internally figures out primary key and security key values
**
** Be aware - if the record doesn't have its security key values set (say it's a
** partial record as part of an update), then those values won't be in the
** security key map... This should probably be considered a bug.
*******************************************************************************/
public static void appendToInput(AuditInput auditInput, QTableMetaData table, QRecord record, String auditMessage)
{
appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), auditMessage);
}
/*******************************************************************************
** Add 1 auditSingleInput to an AuditInput object - with no details (child records).
*******************************************************************************/
public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
public static void appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
{
return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null));
appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null);
}
@ -139,6 +155,44 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
/*******************************************************************************
** For a given record, from a given table, build a map of the record's security
** key values.
**
** If, in case, the record has null value(s), and the oldRecord is given (e.g.,
** for the case of an update, where the record may not have all fields set, and
** oldRecord should be known for doing field-diffs), then try to get the value(s)
** from oldRecord.
**
** Currently, will leave values null if they aren't found after that.
**
** An alternative could be to re-fetch the record from its source if needed...
*******************************************************************************/
public static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional<QRecord> oldRecord)
{
Map<String, Serializable> securityKeyValues = new HashMap<>();
for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
{
Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
if(keyValue == null && oldRecord.isPresent())
{
LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
}
if(keyValue == null)
{
LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
}
securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
}
return securityKeyValues;
}
/*******************************************************************************
**
*******************************************************************************/

397
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java
View File

@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.actions.audits;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@ -52,13 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import static com.kingsrook.qqq.backend.core.actions.audits.AuditAction.getRecordSecurityKeyValues;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -70,6 +71,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
{
private static final QLogger LOG = QLogger.getLogger(DMLAuditAction.class);
public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
/*******************************************************************************
@ -99,34 +102,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
return (output);
}
String contextSuffix = "";
if(StringUtils.hasContent(input.getAuditContext()))
{
contextSuffix = " " + input.getAuditContext();
}
Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
{
String processName = runProcessInput.getProcessName();
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
if(process != null)
{
contextSuffix = " during process: " + process.getLabel();
}
}
QSession qSession = QContext.getQSession();
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
{
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
{
apiLabel = "API";
}
contextSuffix += (" via " + apiLabel + " Version: " + apiVersion);
}
String contextSuffix = getContentSuffix(input);
AuditInput auditInput = new AuditInput();
if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields))
@ -137,7 +113,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : recordList)
{
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix);
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), "Record was " + dmlType.pastTenseVerb + contextSuffix);
}
}
else if(auditLevel.equals(AuditLevel.FIELD))
@ -147,7 +123,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
///////////////////////////////////////////////////////////////////
// do many audits, all with field level details, for FIELD level //
///////////////////////////////////////////////////////////////////
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), qSession);
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
qPossibleValueTranslator.translatePossibleValuesInRecords(table, CollectionUtils.mergeLists(recordList, oldRecordList));
//////////////////////////////////////////
@ -169,92 +145,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
List<QRecord> details = new ArrayList<>();
for(String fieldName : sortedFieldNames)
{
if(!record.getValues().containsKey(fieldName))
{
////////////////////////////////////////////////////////////////////////////////////////////////
// if the stored record doesn't have this field name, then don't audit anything about it //
// this is to deal with our Patch style updates not looking like every field was cleared out. //
////////////////////////////////////////////////////////////////////////////////////////////////
continue;
}
if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
{
continue;
}
QFieldMetaData field = table.getField(fieldName);
Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
QRecord detailRecord = null;
if(oldRecord == null)
{
if(DMLType.INSERT.equals(dmlType) && value == null)
{
continue;
}
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
detailRecord.withValue("newValue", formattedValue);
}
}
else
{
if(!Objects.equals(oldValue, value))
{
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
}
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("newValue", formattedValue);
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
detailRecord.withValue("oldValue", formattedOldValue);
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("oldValue", formattedOldValue);
detailRecord.withValue("newValue", formattedValue);
}
}
}
}
if(detailRecord != null)
{
detailRecord.withValue("fieldName", fieldName);
details.add(detailRecord);
}
makeAuditDetailRecordForField(fieldName, table, dmlType, record, oldRecord)
.ifPresent(details::add);
}
if(details.isEmpty() && DMLType.UPDATE.equals(dmlType))
@ -264,7 +156,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
}
else
{
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.ofNullable(oldRecord)), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
}
}
}
@ -284,6 +176,256 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
static String getContentSuffix(DMLAuditInput input)
{
StringBuilder contextSuffix = new StringBuilder();
/////////////////////////////////////////////////////////////////////////////
// start with context from the input wrapper //
// note, these contexts get propagated down from Input/Update/Delete Input //
/////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(input.getAuditContext()))
{
contextSuffix.append(" ").append(input.getAuditContext());
}
/////////////////////////////////////////////////////////////////////////////////////
// note process label (and a possible context from the process's state) if present //
/////////////////////////////////////////////////////////////////////////////////////
Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
{
String processAuditContext = ValueUtils.getValueAsString(runProcessInput.getValue(AUDIT_CONTEXT_FIELD_NAME));
if(StringUtils.hasContent(processAuditContext))
{
contextSuffix.append(" ").append(processAuditContext);
}
String processName = runProcessInput.getProcessName();
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
if(process != null)
{
contextSuffix.append(" during process: ").append(process.getLabel());
}
}
///////////////////////////////////////////////////
// use api label & version if present in session //
///////////////////////////////////////////////////
QSession qSession = QContext.getQSession();
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
{
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
{
apiLabel = "API";
}
contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
}
return (contextSuffix.toString());
}
/*******************************************************************************
**
*******************************************************************************/
static Optional<QRecord> makeAuditDetailRecordForField(String fieldName, QTableMetaData table, DMLType dmlType, QRecord record, QRecord oldRecord)
{
if(!record.getValues().containsKey(fieldName))
{
////////////////////////////////////////////////////////////////////////////////////////////////
// if the stored record doesn't have this field name, then don't audit anything about it //
// this is to deal with our Patch style updates not looking like every field was cleared out. //
////////////////////////////////////////////////////////////////////////////////////////////////
return (Optional.empty());
}
if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
{
return (Optional.empty());
}
QFieldMetaData field = table.getField(fieldName);
Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
QRecord detailRecord = null;
if(oldRecord == null)
{
if(DMLType.INSERT.equals(dmlType) && value == null)
{
return (Optional.empty());
}
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
detailRecord.withValue("newValue", formattedValue);
}
}
else
{
if(areValuesDifferentForAudit(field, value, oldValue))
{
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
}
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("newValue", formattedValue);
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
detailRecord.withValue("oldValue", formattedOldValue);
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("oldValue", formattedOldValue);
detailRecord.withValue("newValue", formattedValue);
}
}
}
}
if(detailRecord != null)
{
////////////////////////////////////////////////////////////////////
// useful if doing dev in here - but overkill for any other time. //
////////////////////////////////////////////////////////////////////
// LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
detailRecord.withValue("fieldName", fieldName);
return (Optional.of(detailRecord));
}
return (Optional.empty());
}
/*******************************************************************************
**
*******************************************************************************/
static boolean areValuesDifferentForAudit(QFieldMetaData field, Serializable value, Serializable oldValue)
{
try
{
///////////////////
// decimal rules //
///////////////////
if(field.getType().equals(QFieldType.DECIMAL))
{
BigDecimal newBD = ValueUtils.getValueAsBigDecimal(value);
BigDecimal oldBD = ValueUtils.getValueAsBigDecimal(oldValue);
if(newBD == null && oldBD == null)
{
return (false);
}
if(newBD == null || oldBD == null)
{
return (true);
}
return (newBD.compareTo(oldBD) != 0);
}
////////////////////
// dateTime rules //
////////////////////
if(field.getType().equals(QFieldType.DATE_TIME))
{
Instant newI = ValueUtils.getValueAsInstant(value);
Instant oldI = ValueUtils.getValueAsInstant(oldValue);
if(newI == null && oldI == null)
{
return (false);
}
if(newI == null || oldI == null)
{
return (true);
}
////////////////////////////////
// just compare to the second //
////////////////////////////////
return (newI.truncatedTo(ChronoUnit.SECONDS).compareTo(oldI.truncatedTo(ChronoUnit.SECONDS)) != 0);
}
//////////////////
// string rules //
//////////////////
if(field.getType().isStringLike())
{
String newString = ValueUtils.getValueAsString(value);
String oldString = ValueUtils.getValueAsString(oldValue);
boolean newIsNullOrEmpty = !StringUtils.hasContent(newString);
boolean oldIsNullOrEmpty = !StringUtils.hasContent(oldString);
if(newIsNullOrEmpty && oldIsNullOrEmpty)
{
return (false);
}
if(newIsNullOrEmpty || oldIsNullOrEmpty)
{
return (true);
}
return (newString.compareTo(oldString) != 0);
}
/////////////////////////////////////
// default just use Objects.equals //
/////////////////////////////////////
return !Objects.equals(oldValue, value);
}
catch(Exception e)
{
LOG.debug("Error checking areValuesDifferentForAudit", e, logPair("fieldName", field.getName()), logPair("value", value), logPair("oldValue", oldValue));
}
////////////////////////////////////
// default to something simple... //
////////////////////////////////////
return !Objects.equals(oldValue, value);
}
/*******************************************************************************
**
*******************************************************************************/
@ -373,21 +515,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
{
Map<String, Serializable> securityKeyValues = new HashMap<>();
for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
{
securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
}
return securityKeyValues;
}
/*******************************************************************************
**
*******************************************************************************/
@ -407,7 +534,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
private enum DMLType
enum DMLType
{
INSERT("Inserted", true),
UPDATE("Edited", true),

37
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java
View File

@ -22,9 +22,8 @@
package com.kingsrook.qqq.backend.core.actions.automation;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
@ -90,6 +89,10 @@ public class RecordAutomationStatusUpdater
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Avoid setting records to PENDING_INSERT or PENDING_UPDATE even if they don't have any insert or update automations or triggers //
// such records should go straight to OK status. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(canWeSkipPendingAndGoToOkay(table, automationStatus))
{
automationStatus = AutomationStatus.OK;
@ -121,9 +124,13 @@ public class RecordAutomationStatusUpdater
** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just
** move the status straight to OK.
*******************************************************************************/
private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
{
List<TableAutomationAction> tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>());
List<TableAutomationAction> tableActions = Collections.emptyList();
if(table.getAutomationDetails() != null && table.getAutomationDetails().getActions() != null)
{
tableActions = table.getAutomationDetails().getActions();
}
if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
{
@ -135,6 +142,12 @@ public class RecordAutomationStatusUpdater
{
return (false);
}
////////////////////////////////////////////////////////////////////////////////////////
// if we're going to pending-insert, and there are no insert automations or triggers, //
// then we may skip pending and go to okay. //
////////////////////////////////////////////////////////////////////////////////////////
return (true);
}
else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
{
@ -146,9 +159,21 @@ public class RecordAutomationStatusUpdater
{
return (false);
}
}
return (true);
////////////////////////////////////////////////////////////////////////////////////////
// if we're going to pending-update, and there are no insert automations or triggers, //
// then we may skip pending and go to okay. //
////////////////////////////////////////////////////////////////////////////////////////
return (true);
}
else
{
///////////////////////////////////////////////////////////////////////////////////////////////////////
// if we're going to any other automation status - then we may never "skip pending" and go to okay - //
// because we weren't asked to go to pending! //
///////////////////////////////////////////////////////////////////////////////////////////////////////
return (false);
}
}

48
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java
View File

@ -342,22 +342,9 @@ public class PollingAutomationPerTableRunner implements Runnable
boolean anyActionsFailed = false;
for(TableAutomationAction action : actions)
{
try
boolean hadError = applyActionToRecords(table, records, action);
if(hadError)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
if(CollectionUtils.nullSafeHasContents(matchingQRecords))
{
LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
applyActionToMatchingRecords(table, matchingQRecords, action);
}
}
catch(Exception e)
{
LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
anyActionsFailed = true;
}
}
@ -377,6 +364,37 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
** Run one action over a list of records (if they match the action's filter).
**
** @return hadError - true if an exception was caught; false if all OK.
*******************************************************************************/
protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
{
try
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
if(CollectionUtils.nullSafeHasContents(matchingQRecords))
{
LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
applyActionToMatchingRecords(table, matchingQRecords, action);
}
return (false);
}
catch(Exception e)
{
LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
return (true);
}
}
/*******************************************************************************
** For a given action, and a list of records - return a new list, of the ones
** which match the action's filter (if there is one - if not, then all match).

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java
View File

@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLogg
import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -112,6 +114,11 @@ public class ExecuteCodeAction
context.putAll(input.getInput());
}
//////////////////////////////////////////
// safely always set the deploymentMode //
//////////////////////////////////////////
context.put("deploymentMode", ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getDeploymentMode(), null));
/////////////////////////////////////////////////////////////////////////////////
// set the qCodeExecutor into any context objects which are QCodeExecutorAware //
/////////////////////////////////////////////////////////////////////////////////

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java
View File

@ -244,8 +244,6 @@ public class SearchPossibleValueSourceAction
}
}
queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
// todo - skip & limit as params
queryFilter.setLimit(250);
@ -257,6 +255,9 @@ public class SearchPossibleValueSourceAction
input.getDefaultQueryFilter().addSubFilter(queryFilter);
queryFilter = input.getDefaultQueryFilter();
}
queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
queryInput.setFilter(queryFilter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java
View File

@ -84,7 +84,7 @@ public class QContext
actionStackThreadLocal.get().add(actionInput);
}
if(!qInstance.getHasBeenValidated())
if(qInstance != null && !qInstance.getHasBeenValidated())
{
try
{

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java
View File

@ -347,7 +347,7 @@ public class QMetaDataVariableInterpreter
if(canParseAsInteger(envValue))
{
LOG.info("Read env var [" + environmentVariableName + "] as integer " + environmentVariableName);
return (Integer.parseInt(propertyValue));
return (Integer.parseInt(envValue));
}
else
{

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/audits/AuditsMetaDataProvider.java
View File

@ -116,16 +116,19 @@ public class AuditsMetaDataProvider
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(TABLE_NAME_AUDIT_TABLE)
.withTableName(TABLE_NAME_AUDIT_TABLE)
.withOrderByField("name")
);
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(TABLE_NAME_AUDIT_USER)
.withTableName(TABLE_NAME_AUDIT_USER)
.withOrderByField("name")
);
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(TABLE_NAME_AUDIT)
.withTableName(TABLE_NAME_AUDIT)
.withOrderByField("id", false)
);
}

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java
View File

@ -93,6 +93,7 @@ public class QInstance
private Map<String, QSupplementalInstanceMetaData> supplementalMetaData = new LinkedHashMap<>();
private String deploymentMode;
private Map<String, String> environmentValues = new LinkedHashMap<>();
private String defaultTimeZoneId = "UTC";
@ -1165,4 +1166,36 @@ public class QInstance
}
this.joinGraph = joinGraph;
}
/*******************************************************************************
** Getter for deploymentMode
*******************************************************************************/
public String getDeploymentMode()
{
return (this.deploymentMode);
}
/*******************************************************************************
** Setter for deploymentMode
*******************************************************************************/
public void setDeploymentMode(String deploymentMode)
{
this.deploymentMode = deploymentMode;
}
/*******************************************************************************
** Fluent setter for deploymentMode
*******************************************************************************/
public QInstance withDeploymentMode(String deploymentMode)
{
this.deploymentMode = deploymentMode;
return (this);
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/authentication/Auth0AuthenticationMetaData.java
View File

@ -60,7 +60,6 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
private String auth0ClientSecretField;
private Serializable qqqRecordIdField;
/////////////////////////////////////
// fields on the accessToken table //
/////////////////////////////////////

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java
View File

@ -187,7 +187,8 @@ public class QueryStatMetaDataProvider
return (new QPossibleValueSource()
.withType(QPossibleValueSourceType.TABLE)
.withName(QueryStat.TABLE_NAME)
.withTableName(QueryStat.TABLE_NAME));
.withTableName(QueryStat.TABLE_NAME))
.withOrderByField("id", false);
}
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedfilters/SavedFiltersMetaDataProvider.java
View File

@ -88,7 +88,8 @@ public class SavedFiltersMetaDataProvider
.withName(SavedFilter.TABLE_NAME)
.withType(QPossibleValueSourceType.TABLE)
.withTableName(SavedFilter.TABLE_NAME)
.withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);
.withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
.withOrderByField("label");
}
}

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java
View File

@ -303,19 +303,24 @@ public class ScriptsMetaDataProvider
{
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(Script.TABLE_NAME)
.withTableName(Script.TABLE_NAME));
.withTableName(Script.TABLE_NAME)
.withOrderByField("name"));
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(ScriptRevision.TABLE_NAME)
.withTableName(ScriptRevision.TABLE_NAME));
.withTableName(ScriptRevision.TABLE_NAME)
.withOrderByField("scriptId")
.withOrderByField("sequenceNo", false));
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(ScriptType.TABLE_NAME)
.withTableName(ScriptType.TABLE_NAME));
.withTableName(ScriptType.TABLE_NAME)
.withOrderByField("name"));
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(ScriptLog.TABLE_NAME)
.withTableName(ScriptLog.TABLE_NAME));
.withTableName(ScriptLog.TABLE_NAME)
.withOrderByField("id", false));
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(ScriptTypeFileMode.NAME)
@ -378,7 +383,7 @@ public class ScriptsMetaDataProvider
/*******************************************************************************
**
*******************************************************************************/
private QTableMetaData defineTableTriggerTable(String backendName) throws QException
public QTableMetaData defineTableTriggerTable(String backendName) throws QException
{
QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
.withRecordLabelFields("id")

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java
View File

@ -127,7 +127,8 @@ public class QQQTablesMetaDataProvider
return (new QPossibleValueSource()
.withType(QPossibleValueSourceType.TABLE)
.withName(QQQTable.TABLE_NAME)
.withTableName(QQQTable.TABLE_NAME));
.withTableName(QQQTable.TABLE_NAME))
.withOrderByField("label");
}
}

439
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java
View File

@ -33,7 +33,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import com.auth0.client.auth.AuthAPI;
import com.auth0.exception.Auth0Exception;
import com.auth0.json.auth.TokenHolder;
@ -52,6 +51,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.CapturedContext;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.AccessTokenException;
import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
@ -68,11 +68,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.Auth0AuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.model.UserSession;
import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.SimpleStateKey;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
@ -80,7 +80,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.http.HttpStatus;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
@ -90,9 +89,23 @@ import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** QQQ AuthenticationModule for working with Auth0.
**
** createSession can be called with the following fields in its context:
**
** System-User session use-case:
** 1: Takes in an "accessToken" (but doesn't store a userSession record).
** 1b: legacy frontend use-case does the same as system-user!
**
** Web User session use-cases:
** 2: creates a new session (userSession record) by taking an "accessToken"
** 3: looks up an existing session (userSession record) by taking a "sessionUUID"
** 4: takes an "apiKey" (looked up in metaData.AccessTokenTableName - refreshing accessToken with auth0 if needed).
** 5: takes a "basicAuthString" (encoded username:password), which make a new accessToken in auth0
**
*******************************************************************************/
public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
@ -104,14 +117,17 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
public static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 1800;
public static final String AUTH0_ACCESS_TOKEN_KEY = "sessionId";
public static final String API_KEY = "apiKey";
public static final String BASIC_AUTH_KEY = "basicAuthString";
public static final String ACCESS_TOKEN_KEY = "accessToken";
public static final String API_KEY = "apiKey"; // todo - look for users of this, see if we can change to use this constant; maybe move constants up?
public static final String SESSION_UUID_KEY = "sessionUUID";
public static final String BASIC_AUTH_KEY = "basicAuthString"; // todo - look for users of this, see if we can change to use this constant; maybe move constants up?
public static final String TOKEN_NOT_PROVIDED_ERROR = "Access Token was not provided";
public static final String COULD_NOT_DECODE_ERROR = "Unable to decode access token";
public static final String EXPIRED_TOKEN_ERROR = "Token has expired";
public static final String INVALID_TOKEN_ERROR = "An invalid token was provided";
public static final String DO_STORE_USER_SESSION_KEY = "doStoreUserSession";
static final String TOKEN_NOT_PROVIDED_ERROR = "Access Token was not provided";
static final String COULD_NOT_DECODE_ERROR = "Unable to decode access token";
static final String EXPIRED_TOKEN_ERROR = "Token has expired";
static final String INVALID_TOKEN_ERROR = "An invalid token was provided";
////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -149,94 +165,121 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
@Override
public QSession createSession(QInstance qInstance, Map<String, String> context) throws QAuthenticationException
{
Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
///////////////////////////////////////////////////////////
// check if we are processing a Basic Auth Session first //
///////////////////////////////////////////////////////////
if(context.containsKey(BASIC_AUTH_KEY))
{
AuthAPI auth = AuthAPI.newBuilder(metaData.getBaseUrl(), metaData.getClientId(), metaData.getClientSecret()).build();
try
{
/////////////////////////////////////////////////
// decode the credentials from the header auth //
/////////////////////////////////////////////////
String base64Credentials = context.get(BASIC_AUTH_KEY).trim();
String accessToken = getAccessTokenFromBase64BasicAuthCredentials(metaData, auth, base64Credentials);
context.put(AUTH0_ACCESS_TOKEN_KEY, accessToken);
}
catch(Auth0Exception e)
{
////////////////
// ¯\_(ツ)_/¯ //
////////////////
String message = "Error handling basic authentication: " + e.getMessage();
LOG.error(message, e);
throw (new QAuthenticationException(message));
}
}
////////////////////////////////////////////////////////////////////
// get the jwt id or qqq translated token from the context object //
////////////////////////////////////////////////////////////////////
String accessToken = context.get(AUTH0_ACCESS_TOKEN_KEY);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// check to see if the session id is a UUID, if so, that means we need to look up the 'actual' token in the access_token table //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(accessToken != null && StringUtils.isUUID(accessToken))
{
accessToken = lookupActualAccessToken(metaData, accessToken);
}
////////////////////////////////////////////////////////
// if access token is still null, look for an api key //
////////////////////////////////////////////////////////
if(accessToken == null)
{
String apiKey = context.get(API_KEY);
if(apiKey != null)
{
accessToken = getAccessTokenFromApiKey(metaData, apiKey);
}
}
if(accessToken == null)
{
LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
}
//////////////////////////////////////////////////////////////////////////////////////
// decode the token locally to make sure it is valid and to look at when it expires //
//////////////////////////////////////////////////////////////////////////////////////
try
{
Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
String accessToken = null;
if(CollectionUtils.containsKeyWithNonNullValue(context, SESSION_UUID_KEY))
{
/////////////////////////////////////////////////////////////////////////////////////////
// process a sessionUUID - looks up userSession record - cannot create token this way. //
/////////////////////////////////////////////////////////////////////////////////////////
String sessionUUID = context.get(SESSION_UUID_KEY);
LOG.info("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID)));
if(sessionUUID != null)
{
accessToken = getAccessTokenFromSessionUUID(metaData, sessionUUID);
}
}
else if(CollectionUtils.containsKeyWithNonNullValue(context, ACCESS_TOKEN_KEY))
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the context contains an access token, then create a new session based on that token. //
// todo#authHeader - this else/if should maybe be first, but while we have frontend passing both, we want it second //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
accessToken = context.get(ACCESS_TOKEN_KEY);
QSession qSession = buildAndValidateSession(qInstance, accessToken);
////////////////////////////////////////////////////////////////
// build & store userSession db record, if requested to do so //
////////////////////////////////////////////////////////////////
if(CollectionUtils.containsKeyWithNonNullValue(context, DO_STORE_USER_SESSION_KEY))
{
insertUserSession(qInstance, accessToken, qSession);
LOG.info("Creating session based on input accessToken and creating a userSession", logPair("userId", qSession.getUser().getIdReference()));
}
else
{
///////////////////////////////////////////////
// todo#authHeader - remove all this logging //
///////////////////////////////////////////////
String userName = qSession.getUser() != null ? qSession.getUser().getFullName() : null;
if(userName != null && !userName.contains("System User"))
{
LOG.info("Creating session based on input accessToken but not creating a userSession", logPair("userName", qSession.getUser().getFullName()));
}
}
return (qSession);
}
else if(CollectionUtils.containsKeyWithNonNullValue(context, BASIC_AUTH_KEY))
{
//////////////////////////////////////////////////////////////////////////////////////
// Process a basic auth (username:password) //
// by getting an access token from auth0 (re-using from state provider if possible) //
//////////////////////////////////////////////////////////////////////////////////////
AuthAPI auth = AuthAPI.newBuilder(metaData.getBaseUrl(), metaData.getClientId(), metaData.getClientSecret()).build();
try
{
/////////////////////////////////////////////////
// decode the credentials from the header auth //
/////////////////////////////////////////////////
String base64Credentials = context.get(BASIC_AUTH_KEY).trim();
LOG.info("Creating session from basicAuthentication", logPair("base64Credentials", maskForLog(base64Credentials)));
accessToken = getAccessTokenFromBase64BasicAuthCredentials(metaData, auth, base64Credentials);
}
catch(Auth0Exception e)
{
////////////////
// ¯\_(ツ)_/¯ //
////////////////
String message = "Error handling basic authentication: " + e.getMessage();
LOG.error(message, e);
throw (new QAuthenticationException(message));
}
}
else if(CollectionUtils.containsKeyWithNonNullValue(context, API_KEY))
{
///////////////////////////////////////////////////////////////////////////////////////
// process an api key - looks up client application token (creating token if needed) //
///////////////////////////////////////////////////////////////////////////////////////
String apiKey = context.get(API_KEY);
LOG.info("Creating session from apiKey (accessTokenTable)", logPair("apiKey", maskForLog(apiKey)));
if(apiKey != null)
{
accessToken = getAccessTokenFromApiKey(metaData, apiKey);
}
}
/* todo confirm this is deprecated
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// check to see if the session id is a UUID, if so, that means we need to look up the 'actual' token in the access_token table //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(accessToken != null && StringUtils.isUUID(accessToken))
{
accessToken = lookupActualAccessToken(metaData, accessToken);
}
*/
///////////////////////////////////////////
// if token wasn't found by now, give up //
///////////////////////////////////////////
if(accessToken == null)
{
LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
}
/////////////////////////////////////////////////////
// try to build session to see if still valid //
// then call method to check more session validity //
/////////////////////////////////////////////////////
QSession qSession = buildQSessionFromToken(accessToken, qInstance);
if(isSessionValid(qInstance, qSession))
{
return (qSession);
}
///////////////////////////////////////////////////////////////////////////////////////
// if we make it here it means we have never validated this token or its been a long //
// enough duration so we need to re-verify the token //
///////////////////////////////////////////////////////////////////////////////////////
qSession = revalidateTokenAndBuildSession(qInstance, accessToken);
////////////////////////////////////////////////////////////////////
// put now into state so we dont check until next interval passes //
///////////////////////////////////////////////////////////////////
StateProviderInterface spi = getStateProvider();
SimpleStateKey<String> key = new SimpleStateKey<>(qSession.getIdReference());
spi.put(key, Instant.now());
return (qSession);
return buildAndValidateSession(qInstance, accessToken);
}
catch(QAuthenticationException qae)
{
throw (qae);
}
catch(JWTDecodeException jde)
{
@ -272,6 +315,61 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
** Insert a session as a new record into userSession table
*******************************************************************************/
private void insertUserSession(QInstance qInstance, String accessToken, QSession qSession) throws QException
{
CapturedContext capturedContext = QContext.capture();
try
{
QContext.init(qInstance, null);
QContext.setQSession(getChickenAndEggSession());
UserSession userSession = new UserSession()
.withUuid(qSession.getUuid())
.withUserId(qSession.getUser().getIdReference())
.withAccessToken(accessToken);
new InsertAction().execute(new InsertInput(UserSession.TABLE_NAME).withRecordEntity(userSession));
}
finally
{
QContext.init(capturedContext);
}
}
/*******************************************************************************
**
*******************************************************************************/
private QSession buildAndValidateSession(QInstance qInstance, String accessToken) throws JwkException
{
QSession qSession = buildQSessionFromToken(accessToken, qInstance);
if(isSessionValid(qInstance, qSession))
{
return (qSession);
}
//////////////////////////////////////////////////////////////////////////////////////////
// if we make it here it means we have never validated this token or it has been a long //
// enough duration so we need to re-verify the token //
//////////////////////////////////////////////////////////////////////////////////////////
qSession = revalidateTokenAndBuildSession(qInstance, accessToken);
/////////////////////////////////////////////////////////////////////
// put now into state so we don't check until next interval passes //
/////////////////////////////////////////////////////////////////////
StateProviderInterface spi = getStateProvider();
SimpleStateKey<String> key = new SimpleStateKey<>(qSession.getIdReference());
spi.put(key, Instant.now());
return (qSession);
}
/*******************************************************************************
**
*******************************************************************************/
@ -299,7 +397,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
byte[] credDecoded = Base64.getDecoder().decode(base64Credentials);
String credentials = new String(credDecoded, StandardCharsets.UTF_8);
String accessToken = getAccessTokenFromAuth0(metaData, auth, credentials);
String accessToken = getAccessTokenForUsernameAndPasswordFromAuth0(metaData, auth, credentials);
stateProvider.put(accessTokenStateKey, accessToken);
stateProvider.put(timestampStateKey, Instant.now());
return (accessToken);
@ -310,7 +408,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
**
*******************************************************************************/
protected String getAccessTokenFromAuth0(Auth0AuthenticationMetaData metaData, AuthAPI auth, String credentials) throws Auth0Exception
protected String getAccessTokenForUsernameAndPasswordFromAuth0(Auth0AuthenticationMetaData metaData, AuthAPI auth, String credentials) throws Auth0Exception
{
/////////////////////////////////////
// call auth0 with a login request //
@ -620,75 +718,11 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
** create a new auth0 access token
** make http request to Auth0 for a new access token for an application - e.g.,
** with a clientId and clientSecret as params
**
*******************************************************************************/
public String createAccessToken(QAuthenticationMetaData metaData, String clientId, String clientSecret) throws AccessTokenException
{
QSession sessionBefore = QContext.getQSession();
Auth0AuthenticationMetaData auth0MetaData = (Auth0AuthenticationMetaData) metaData;
try
{
QContext.setQSession(getChickenAndEggSession());
///////////////////////////////////////////////////////////////////////////////////////
// fetch the application from database, will throw accesstokenexception if not found //
///////////////////////////////////////////////////////////////////////////////////////
QRecord clientAuth0Application = getClientAuth0Application(auth0MetaData, clientId);
/////////////////////////////////////////////////////////////////////////////////////////////////
// request access token from auth0 if exception is not thrown, that means 200OK, we want to //
// store the actual access token in the database, and return a unique value //
// back to the user which will be what they use on subseqeunt requests (because token too big) //
/////////////////////////////////////////////////////////////////////////////////////////////////
JSONObject accessTokenData = requestAccessTokenFromAuth0(auth0MetaData, clientId, clientSecret);
Integer expiresInSeconds = accessTokenData.getInt("expires_in");
String accessToken = accessTokenData.getString("access_token");
String uuid = UUID.randomUUID().toString();
/////////////////////////////////
// store the details in the db //
/////////////////////////////////
QRecord accessTokenRecord = new QRecord()
.withValue(auth0MetaData.getClientAuth0ApplicationIdField(), clientAuth0Application.getValue("id"))
.withValue(auth0MetaData.getAuth0AccessTokenField(), accessToken)
.withValue(auth0MetaData.getQqqAccessTokenField(), uuid)
.withValue(auth0MetaData.getExpiresInSecondsField(), expiresInSeconds);
InsertInput input = new InsertInput();
input.setTableName(auth0MetaData.getAccessTokenTableName());
input.setRecords(List.of(accessTokenRecord));
new InsertAction().execute(input);
//////////////////////////////////
// update and send the response //
//////////////////////////////////
accessTokenData.put("access_token", uuid);
accessTokenData.remove("scope");
return (accessTokenData.toString());
}
catch(AccessTokenException ate)
{
throw (ate);
}
catch(Exception e)
{
throw (new AccessTokenException(e.getMessage(), e));
}
finally
{
QContext.setQSession(sessionBefore);
}
}
/*******************************************************************************
** make http request to Auth0 for a new access token
**
*******************************************************************************/
public JSONObject requestAccessTokenFromAuth0(Auth0AuthenticationMetaData auth0MetaData, String clientId, String clientSecret) throws AccessTokenException
public JSONObject requestAccessTokenForClientIdAndSecretFromAuth0(Auth0AuthenticationMetaData auth0MetaData, String clientId, String clientSecret) throws AccessTokenException
{
///////////////////////////////////////////////////////////////////
// make a request to Auth0 using the client_id and client_secret //
@ -776,6 +810,63 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
** Look up access_token from session UUID
**
*******************************************************************************/
private String getAccessTokenFromSessionUUID(Auth0AuthenticationMetaData metaData, String sessionUUID) throws QAuthenticationException
{
String accessToken = null;
QSession beforeSession = QContext.getQSession();
try
{
QContext.setQSession(getChickenAndEggSession());
///////////////////////////////////////
// query for the user session record //
///////////////////////////////////////
QRecord userSessionRecord = new GetAction().executeForRecord(new GetInput(UserSession.TABLE_NAME)
.withUniqueKey(Map.of("uuid", sessionUUID))
.withShouldMaskPasswords(false)
.withShouldOmitHiddenFields(false));
if(userSessionRecord != null)
{
accessToken = userSessionRecord.getValueString("accessToken");
////////////////////////////////////////////////////////////
// decode the accessToken and make sure it is not expired //
////////////////////////////////////////////////////////////
if(accessToken != null)
{
DecodedJWT jwt = JWT.decode(accessToken);
if(jwt.getExpiresAtAsInstant().isBefore(Instant.now()))
{
throw (new QAuthenticationException("accessToken is expired"));
}
}
}
}
catch(QAuthenticationException qae)
{
throw (qae);
}
catch(Exception e)
{
LOG.warn("Error looking up userSession by sessionUUID", e);
throw (new QAuthenticationException("Error looking up userSession by sessionUUID", e));
}
finally
{
QContext.setQSession(beforeSession);
}
return (accessToken);
}
/*******************************************************************************
** Look up access_token from api key
**
@ -841,7 +932,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// store the actual access token in the database, and return a unique value //
// back to the user which will be what they use on subsequent requests (because token too big) //
/////////////////////////////////////////////////////////////////////////////////////////////////
JSONObject accessTokenData = requestAccessTokenFromAuth0(metaData, clientId, clientSecret);
JSONObject accessTokenData = requestAccessTokenForClientIdAndSecretFromAuth0(metaData, clientId, clientSecret);
Integer expiresInSeconds = accessTokenData.getInt("expires_in");
accessToken = accessTokenData.getString("access_token");
@ -872,25 +963,23 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
** Look up client_auth0_application record, return if found.
**
*******************************************************************************/
QRecord getClientAuth0Application(Auth0AuthenticationMetaData metaData, String clientId) throws QException
static String maskForLog(String input)
{
//////////////////////////////////////////////////////////////////////////////////////
// try to look up existing auth0 application from database, insert one if not found //
//////////////////////////////////////////////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(metaData.getClientAuth0ApplicationTableName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(metaData.getAuth0ClientIdField(), QCriteriaOperator.EQUALS, clientId)));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
if(input == null)
{
return (queryOutput.getRecords().get(0));
return (null);
}
throw (new AccessTokenException("This client has not been configured to use the API.", HttpStatus.SC_UNAUTHORIZED));
if(input.length() < 8)
{
return ("******");
}
else
{
return (input.substring(0, 6) + "******");
}
}
}

73
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/metadata/UserSessionMetaDataProducer.java Normal file
View File

@ -0,0 +1,73 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.modules.authentication.implementations.metadata;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.model.UserSession;
/*******************************************************************************
** Meta Data Producer for UserSession
*******************************************************************************/
public class UserSessionMetaDataProducer extends MetaDataProducer<QTableMetaData>
{
private final String backendName;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public UserSessionMetaDataProducer(String backendName)
{
this.backendName = backendName;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QTableMetaData produce(QInstance qInstance) throws QException
{
QTableMetaData tableMetaData = new QTableMetaData()
.withName(UserSession.TABLE_NAME)
.withBackendName(backendName)
.withRecordLabelFormat("%s")
.withRecordLabelFields("id")
.withPrimaryKeyField("id")
.withUniqueKey(new UniqueKey("uuid"))
.withFieldsFromEntity(UserSession.class)
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE));
return tableMetaData;
}
}

262
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/model/UserSession.java Normal file
View File

@ -0,0 +1,262 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.modules.authentication.implementations.model;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
/*******************************************************************************
** QRecord Entity for UserSession table
*******************************************************************************/
public class UserSession extends QRecordEntity
{
public static final String TABLE_NAME = "userSession";
@QField(isEditable = false)
private Integer id;
@QField(isEditable = false)
private Instant createDate;
@QField(isEditable = false)
private Instant modifyDate;
@QField(isEditable = false, isHidden = true, maxLength = 40, valueTooLongBehavior = ValueTooLongBehavior.ERROR)
private String uuid;
@QField(isEditable = false, isHidden = true)
private String accessToken;
@QField(isEditable = false, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS)
private String userId;
/*******************************************************************************
** Default constructor
*******************************************************************************/
public UserSession()
{
}
/*******************************************************************************
** Constructor that takes a QRecord
*******************************************************************************/
public UserSession(QRecord record)
{
populateFromQRecord(record);
}
/*******************************************************************************
** Getter for id
*******************************************************************************/
public Integer getId()
{
return (this.id);
}
/*******************************************************************************
** Setter for id
*******************************************************************************/
public void setId(Integer id)
{
this.id = id;
}
/*******************************************************************************
** Fluent setter for id
*******************************************************************************/
public UserSession withId(Integer id)
{
this.id = id;
return (this);
}
/*******************************************************************************
** Getter for createDate
*******************************************************************************/
public Instant getCreateDate()
{
return (this.createDate);
}
/*******************************************************************************
** Setter for createDate
*******************************************************************************/
public void setCreateDate(Instant createDate)
{
this.createDate = createDate;
}
/*******************************************************************************
** Fluent setter for createDate
*******************************************************************************/
public UserSession withCreateDate(Instant createDate)
{
this.createDate = createDate;
return (this);
}
/*******************************************************************************
** Getter for modifyDate
*******************************************************************************/
public Instant getModifyDate()
{
return (this.modifyDate);
}
/*******************************************************************************
** Setter for modifyDate
*******************************************************************************/
public void setModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
}
/*******************************************************************************
** Fluent setter for modifyDate
*******************************************************************************/
public UserSession withModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
return (this);
}
/*******************************************************************************
** Getter for uuid
*******************************************************************************/
public String getUuid()
{
return (this.uuid);
}
/*******************************************************************************
** Setter for uuid
*******************************************************************************/
public void setUuid(String uuid)
{
this.uuid = uuid;
}
/*******************************************************************************
** Fluent setter for uuid
*******************************************************************************/
public UserSession withUuid(String uuid)
{
this.uuid = uuid;
return (this);
}
/*******************************************************************************
** Getter for accessToken
*******************************************************************************/
public String getAccessToken()
{
return (this.accessToken);
}
/*******************************************************************************
** Setter for accessToken
*******************************************************************************/
public void setAccessToken(String accessToken)
{
this.accessToken = accessToken;
}
/*******************************************************************************
** Fluent setter for accessToken
*******************************************************************************/
public UserSession withAccessToken(String accessToken)
{
this.accessToken = accessToken;
return (this);
}
/*******************************************************************************
** Getter for userId
*******************************************************************************/
public String getUserId()
{
return (this.userId);
}
/*******************************************************************************
** Setter for userId
*******************************************************************************/
public void setUserId(String userId)
{
this.userId = userId;
}
/*******************************************************************************
** Fluent setter for userId
*******************************************************************************/
public UserSession withUserId(String userId)
{
this.userId = userId;
return (this);
}
}

53
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java
View File

@ -25,11 +25,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryFilterLink;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@ -42,7 +46,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.scripts.Script;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptLog;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
@ -121,16 +127,25 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
getInput.setTableName(Script.TABLE_NAME);
getInput.setPrimaryKey(scriptId);
GetOutput getOutput = new GetAction().execute(getInput);
if(getOutput.getRecord() == null)
QRecord script = getOutput.getRecord();
if(script == null)
{
throw (new QException("Could not find script by id: " + scriptId));
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set an "audit context" - so any DML executed during the script will include the note of what script was running. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"" + script.getValue("name") + "\"");
String tableName = script.getValueString("tableName");
RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
input.setRecordList(runBackendStepInput.getRecords());
input.setCodeReference(new AdHocScriptCodeReference().withScriptId(scriptId));
input.setTableName(getOutput.getRecord().getValueString("tableName"));
input.setTableName(tableName);
input.setLogger(scriptLogger);
RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
Exception caughtException = null;
try
@ -147,11 +162,17 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
caughtException = e;
}
String auditMessage = "Script \"" + script.getValueString("name") + "\" (id: " + scriptId + ") was executed against this record";
//////////////////////////////////////////////////////////
// add the record to the appropriate processSummaryLine //
//////////////////////////////////////////////////////////
if(scriptLogger.getScriptLog() != null)
{
Integer id = scriptLogger.getScriptLog().getValueInteger("id");
if(id != null)
{
auditMessage += ", creating script log: " + id;
boolean hadError = BooleanUtils.isTrue(scriptLogger.getScriptLog().getValueBoolean("hadError"));
(hadError ? errorScriptLogIds : okScriptLogIds).add(id);
}
@ -160,6 +181,34 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
{
unloggedExceptionLine.incrementCount(runBackendStepInput.getRecords().size());
}
////////////////////////////////////////////////////////////
// audit that the script was executed against the records //
////////////////////////////////////////////////////////////
audit(runBackendStepInput, tableName, auditMessage);
}
/*******************************************************************************
** for each input record, add an audit stating that the script was executed.
*******************************************************************************/
private static void audit(RunBackendStepInput runBackendStepInput, String tableName, String auditMessage)
{
try
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
AuditInput auditInput = new AuditInput();
for(QRecord record : runBackendStepInput.getRecords())
{
AuditAction.appendToInput(auditInput, table, record, auditMessage);
}
new AuditAction().execute(auditInput);
}
catch(Exception e)
{
LOG.warn("Error recording audits after running record script", e, logPair("tableName", tableName), logPair("auditMessage", auditMessage));
}
}

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java
View File

@ -663,4 +663,19 @@ public class CollectionUtils
return (output);
}
/*******************************************************************************
**
*******************************************************************************/
public static <K> boolean containsKeyWithNonNullValue(Map<K, ?> map, K key)
{
if(map == null)
{
return (false);
}
return (map.containsKey(key) && map.get(key) != null);
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java
View File

@ -48,6 +48,8 @@ public class BaseTest
@BeforeEach
void baseBeforeEach()
{
System.setProperty("qqq.logger.logSessionId.disabled", "true");
QContext.init(TestUtils.defineInstance(), new QSession()
.withUser(new QUser()
.withIdReference("001")

29
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java
View File

@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -38,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -197,4 +199,31 @@ class AuditActionTest extends BaseTest
assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail3"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAppendToInputThatTakesRecordNotIdAndSecurityKeyValues()
{
AuditInput auditInput = new AuditInput();
//////////////////////////////////////////////////////////////
// make sure the recordId & securityKey got build correctly //
//////////////////////////////////////////////////////////////
AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47).withValue("storeId", 42), "Test");
AuditSingleInput auditSingleInput = auditInput.getAuditSingleInputList().get(0);
assertEquals(47, auditSingleInput.getRecordId());
assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, 42), auditSingleInput.getSecurityKeyValues());
///////////////////////////////////////////////////////////////////////////////////////////
// acknowledge that we might get back a null key value if the record doesn't have it set //
///////////////////////////////////////////////////////////////////////////////////////////
AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47), "Test");
auditSingleInput = auditInput.getAuditSingleInputList().get(1);
assertEquals(47, auditSingleInput.getRecordId());
assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, null), auditSingleInput.getSecurityKeyValues());
}
}

200
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditActionTest.java
View File

@ -22,12 +22,14 @@
package com.kingsrook.qqq.backend.core.actions.audits;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@ -36,10 +38,18 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.INSERT;
import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.UPDATE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
@ -200,4 +210,194 @@ class DMLAuditActionTest extends BaseTest
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testMakeAuditDetailRecordForField()
{
QTableMetaData table = new QTableMetaData()
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withLabel("Create Date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withLabel("Modify Date"))
.withField(new QFieldMetaData("someTimestamp", QFieldType.DATE_TIME).withLabel("Some Timestamp"))
.withField(new QFieldMetaData("name", QFieldType.STRING).withLabel("Name"))
.withField(new QFieldMetaData("seqNo", QFieldType.INTEGER).withLabel("Sequence No."))
.withField(new QFieldMetaData("price", QFieldType.DECIMAL).withLabel("Price"));
///////////////////////////////
// create date - never audit //
///////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("createDate", table, INSERT,
new QRecord().withValue("createDate", Instant.now()),
new QRecord().withValue("createDate", Instant.now().minusSeconds(100))))
.isEmpty();
///////////////////////////////
// modify date - never audit //
///////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("modifyDate", table, UPDATE,
new QRecord().withValue("modifyDate", Instant.now()),
new QRecord().withValue("modifyDate", Instant.now().minusSeconds(100))))
.isEmpty();
////////////////////////////////////////////////////////
// datetime different only in precision - don't audit //
////////////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")),
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
.isEmpty();
/////////////////////////////////////////////
// datetime actually different - audit it. //
/////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:09Z")),
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Changed Some Timestamp from 2023.* to 2023.*"));
////////////////////////////////////////////////
// datetime changing null to not null - audit //
////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")),
new QRecord().withValue("someTimestamp", null)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Set Some Timestamp to 2023.*"));
////////////////////////////////////////////////
// datetime changing not null to null - audit //
////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", null),
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Removed 2023.*from Some Timestamp"));
////////////////////////////////////////
// string that is the same - no audit //
////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
new QRecord().withValue("name", "Homer"),
new QRecord().withValue("name", "Homer")))
.isEmpty();
//////////////////////////////////////////
// string from null to empty - no audit //
//////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
new QRecord().withValue("name", null),
new QRecord().withValue("name", "")))
.isEmpty();
//////////////////////////////////////////
// string from empty to null - no audit //
//////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
new QRecord().withValue("name", ""),
new QRecord().withValue("name", null)))
.isEmpty();
//////////////////////////////////////////////////////////
// decimal that only changes in precision - don't audit //
//////////////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", "10"),
new QRecord().withValue("price", new BigDecimal("10.00"))))
.isEmpty();
//////////////////////////////////////////////////
// decimal that's actually different - do audit //
//////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", "10.01"),
new QRecord().withValue("price", new BigDecimal("10.00"))))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Changed Price from 10.00 to 10.01"));
///////////////////////////////////////
// decimal null, input "" - no audit //
///////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", ""),
new QRecord().withValue("price", null)))
.isEmpty();
/////////////////////////////////////////
// decimal not-null to null - do audit //
/////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", BigDecimal.ONE),
new QRecord().withValue("price", null)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Set Price to 1"));
/////////////////////////////////////////
// decimal null to not-null - do audit //
/////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", null),
new QRecord().withValue("price", BigDecimal.ONE)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Removed 1 from Price"));
///////////////////////////////////////
// integer null, input "" - no audit //
///////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE,
new QRecord().withValue("seqNo", ""),
new QRecord().withValue("seqNo", null)))
.isEmpty();
////////////////////////////////
// integer changed - do audit //
////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE,
new QRecord().withValue("seqNo", 2),
new QRecord().withValue("seqNo", 1)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Changed Sequence No. from 1 to 2"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetContextSuffix()
{
assertEquals("", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
assertEquals(" while shipping an order", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order")));
QContext.pushAction(new RunProcessInput().withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\""));
assertEquals(" via Script \"My Script\"", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.popAction();
QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE));
assertEquals(" during process: Greet", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.popAction();
QContext.setQSession(new QSession().withValue("apiVersion", "1.0"));
assertEquals(" via API Version: 1.0", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API"));
assertEquals(" via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE).withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\""));
QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API"));
assertEquals(" while shipping an order via Script \"My Script\" during process: Greet via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order")));
QContext.popAction();
}
}

128
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdaterTest.java Normal file
View File

@ -0,0 +1,128 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.automation;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for RecordAutomationStatusUpdater
*******************************************************************************/
class RecordAutomationStatusUpdaterTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCanWeSkipPendingAndGoToOkay() throws QException
{
QContext.getQInstance()
.addTable(new ScriptsMetaDataProvider().defineTableTriggerTable(TestUtils.MEMORY_BACKEND_NAME));
////////////////////////////////////////////////////////////
// define tables with various automations and/or triggers //
////////////////////////////////////////////////////////////
QTableMetaData tableWithNoAutomations = new QTableMetaData()
.withName("tableWithNoAutomations");
QTableMetaData tableWithInsertAutomation = new QTableMetaData()
.withName("tableWithInsertAutomation")
.withAutomationDetails(new QTableAutomationDetails()
.withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT)));
QTableMetaData tableWithUpdateAutomation = new QTableMetaData()
.withName("tableWithUpdateAutomation")
.withAutomationDetails(new QTableAutomationDetails()
.withAction(new TableAutomationAction()
.withTriggerEvent(TriggerEvent.POST_UPDATE)));
QTableMetaData tableWithInsertAndUpdateAutomations = new QTableMetaData()
.withName("tableWithInsertAndUpdateAutomations ")
.withAutomationDetails(new QTableAutomationDetails()
.withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT))
.withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_UPDATE)));
QTableMetaData tableWithInsertTrigger = new QTableMetaData()
.withName("tableWithInsertTrigger");
new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME)
.withRecordEntity(new TableTrigger().withTableName(tableWithInsertTrigger.getName()).withPostInsert(true).withPostUpdate(false)));
QTableMetaData tableWithUpdateTrigger = new QTableMetaData()
.withName("tableWithUpdateTrigger");
new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME)
.withRecordEntity(new TableTrigger().withTableName(tableWithUpdateTrigger.getName()).withPostInsert(false).withPostUpdate(true)));
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// tests for going to PENDING_INSERT. //
// we should be allowed to skip and go to OK (return true) if the table does not have insert automations or triggers //
// we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// tests for going to PENDING_UPDATE. //
// we should be allowed to skip and go to OK (return true) if the table does not have update automations or triggers //
// we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// tests for going to non-PENDING states //
// this function should NEVER return true for skipping pending if the target state (2nd arg) isn't a pending state. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(AutomationStatus automationStatus : List.of(AutomationStatus.RUNNING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS, AutomationStatus.FAILED_INSERT_AUTOMATIONS, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, AutomationStatus.OK))
{
for(QTableMetaData table : List.of(tableWithNoAutomations, tableWithInsertAutomation, tableWithUpdateAutomation, tableWithInsertAndUpdateAutomations, tableWithInsertTrigger, tableWithUpdateTrigger))
{
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(table, automationStatus), "Should never be okay to skip pending and go to OK (because we weren't going to pending). table=[" + table.getName() + "], status=[" + automationStatus + "]");
}
}
}
}

207
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java
View File

@ -22,21 +22,27 @@
package com.kingsrook.qqq.backend.core.actions.automation.polling;
import java.io.Serializable;
import java.time.LocalDate;
import java.time.Month;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -60,7 +66,9 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
/*******************************************************************************
@ -155,6 +163,40 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
/*******************************************************************************
** Test that if an automation has an error that we get error status
*******************************************************************************/
@Test
void testAutomationWithError() throws QException
{
QInstance qInstance = QContext.getQInstance();
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
// insert 2 person records, both updated by the insert action, and 1 logged by logger-on-update automation //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("id", 1).withValue("firstName", "Han").withValue("lastName", "Solo").withValue("birthDate", LocalDate.parse("1977-05-25")),
new QRecord().withValue("id", 2).withValue("firstName", "Luke").withValue("lastName", "Skywalker").withValue("birthDate", LocalDate.parse("1977-05-25")),
new QRecord().withValue("id", 3).withValue("firstName", "Darth").withValue("lastName", "Vader").withValue("birthDate", LocalDate.parse("1977-05-25"))
));
new InsertAction().execute(insertInput);
assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
/////////////////////////
// run the automations //
/////////////////////////
runAllTableActions(qInstance);
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// make sure all records are in status ERROR (even though only 1 threw, it breaks the page that it's in) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
assertAllRecordsAutomationStatus(AutomationStatus.FAILED_INSERT_AUTOMATIONS);
}
/*******************************************************************************
**
*******************************************************************************/
@ -169,7 +211,6 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
// note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
}
}
@ -228,6 +269,77 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
/*******************************************************************************
** Test a large-ish number - to demonstrate paging working - and how it deals
** with intermittent errors
**
*******************************************************************************/
@Test
void testMultiPagesWithSomeFailures() throws QException
{
QInstance qInstance = QContext.getQInstance();
//////////////////////////////////////////////////////////////////////////////////////////////////////
// adjust table's automations batch size - as any exceptions thrown put the whole batch into error. //
// so we'll make batches (pages) of 100 - run for 500 records, and make just a couple bad records //
// that'll cause errors - so we should get a few failed pages, and the rest ok. //
//////////////////////////////////////////////////////////////////////////////////////////////////////
int pageSize = 100;
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
.getAutomationDetails()
.setOverrideBatchSize(pageSize);
//////////////////////////////////////////////////////////////////////////////////
// insert many people - half who should be updated by the AgeChecker automation //
//////////////////////////////////////////////////////////////////////////////////
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(new ArrayList<>());
int SIZE = 500;
for(int i = 0; i < SIZE; i++)
{
insertInput.getRecords().add(new QRecord().withValue("firstName", "Qui Gon").withValue("lastName", "Jinn " + i).withValue("birthDate", LocalDate.now()));
insertInput.getRecords().add(new QRecord().withValue("firstName", "Obi Wan").withValue("lastName", "Kenobi " + i));
/////////////////////////////////
// throw 2 Darths into the mix //
/////////////////////////////////
if(i == 101 || i == 301)
{
insertInput.getRecords().add(new QRecord().withValue("firstName", "Darth").withValue("lastName", "Maul " + i));
}
}
InsertOutput insertOutput = new InsertAction().execute(insertInput);
List<Serializable> insertedIds = insertOutput.getRecords().stream().map(r -> r.getValue("id")).toList();
assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
/////////////////////////
// run the automations //
/////////////////////////
runAllTableActions(qInstance);
////////////////////////////////////////////////////////////////////
// make sure that some records became ok, but others became error //
////////////////////////////////////////////////////////////////////
QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, insertedIds))));
List<QRecord> okRecords = queryOutput.getRecords().stream().filter(r -> AutomationStatus.OK.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList();
List<QRecord> failedRecords = queryOutput.getRecords().stream().filter(r -> AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList();
assertFalse(okRecords.isEmpty(), "Some inserted records should be automation status OK");
assertFalse(failedRecords.isEmpty(), "Some inserted records should be automation status Failed");
assertEquals(insertedIds.size(), okRecords.size() + failedRecords.size(), "All inserted records should be OK or Failed");
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// make sure that only 2 pages failed - meaning our number of failedRecords is < pageSize * 2 (as any page may be smaller than the pageSize) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertThat(failedRecords.size()).isLessThanOrEqualTo(pageSize * 2).describedAs("No more than 2 pages should be in failed status.");
}
/*******************************************************************************
** Test running a process for automation, instead of a code ref.
*******************************************************************************/
@ -367,6 +479,61 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testServerShutdownMidRunLeavesRecordsInRunningStatus() throws QException
{
QInstance qInstance = QContext.getQInstance();
/////////////////////////////////////////////////////////////////////////////
// insert 2 person records that should have insert action ran against them //
/////////////////////////////////////////////////////////////////////////////
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()),
new QRecord().withValue("id", 2).withValue("firstName", "Darin").withValue("birthDate", LocalDate.now())
));
new InsertAction().execute(insertInput);
assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// duplicate the runAllTableActions method - but using the subclass of PollingAutomationPerTableRunner that will throw. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertThatThrownBy(() ->
{
List<PollingAutomationPerTableRunner.TableActions> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions)
{
PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(qInstance, TestUtils.POLLING_AUTOMATION, QSession::new, tableAction);
/////////////////////////////////////////////////////////////////////////////////////////////////////
// note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
}
}).hasMessage(PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun.EXCEPTION_MESSAGE);
//////////////////////////////////////////////////
// records should be "leaked" in running status //
//////////////////////////////////////////////////
assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS);
/////////////////////////////////////
// simulate another run of the job //
/////////////////////////////////////
runAllTableActions(qInstance);
////////////////////////////////////////////////////////////////////////////////
// it should NOT have updated those records - they're officially "leaked" now //
////////////////////////////////////////////////////////////////////////////////
assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS);
}
/*******************************************************************************
**
*******************************************************************************/
@ -376,4 +543,42 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
.isNotEmpty()
.allMatch(r -> pendingInsertAutomations.getId().equals(r.getValue(TestUtils.standardQqqAutomationStatusField().getName())));
}
/*******************************************************************************
** this subclass of the class under test allows us to simulate:
**
** what happens if, after records have been marked as running-updates, if,
** for example, a server shuts down?
**
** It does this by overriding a method that runs between those points in time,
** and throwing a runtime exception.
*******************************************************************************/
public static class PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun extends PollingAutomationPerTableRunner
{
private static String EXCEPTION_MESSAGE = "Throwing outside of catch here, to simulate a server shutdown mid-run";
/*******************************************************************************
**
*******************************************************************************/
public PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActions tableActions)
{
super(instance, providerName, sessionSupplier, tableActions);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
{
throw (new RuntimeException(EXCEPTION_MESSAGE));
}
}
}

46
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java
View File

@ -273,6 +273,52 @@ class QMetaDataVariableInterpreterTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetIntegerFromPropertyOrEnvironment()
{
QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
//////////////////////////////////////////////////////////
// if neither prop nor env is set, get back the default //
//////////////////////////////////////////////////////////
assertEquals(1, interpreter.getIntegerFromPropertyOrEnvironment("notSet", "NOT_SET", 1));
assertEquals(2, interpreter.getIntegerFromPropertyOrEnvironment("notSet", "NOT_SET", 2));
/////////////////////////////////////////////
// unrecognized values are same as not set //
/////////////////////////////////////////////
System.setProperty("unrecognized", "asdf");
interpreter.setEnvironmentOverrides(Map.of("UNRECOGNIZED", "qwerty"));
assertEquals(3, interpreter.getIntegerFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", 3));
assertEquals(4, interpreter.getIntegerFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", 4));
/////////////////////////////////
// if only prop is set, get it //
/////////////////////////////////
assertEquals(5, interpreter.getIntegerFromPropertyOrEnvironment("foo.size", "FOO_SIZE", 5));
System.setProperty("foo.size", "6");
assertEquals(6, interpreter.getIntegerFromPropertyOrEnvironment("foo.size", "FOO_SIZE", 7));
////////////////////////////////
// if only env is set, get it //
////////////////////////////////
assertEquals(8, interpreter.getIntegerFromPropertyOrEnvironment("bar.size", "BAR_SIZE", 8));
interpreter.setEnvironmentOverrides(Map.of("BAR_SIZE", "9"));
assertEquals(9, interpreter.getIntegerFromPropertyOrEnvironment("bar.size", "BAR_SIZE", 10));
///////////////////////////////////
// if both are set, get the prop //
///////////////////////////////////
System.setProperty("baz.size", "11");
interpreter.setEnvironmentOverrides(Map.of("BAZ_SIZE", "12"));
assertEquals(11, interpreter.getIntegerFromPropertyOrEnvironment("baz.size", "BAZ_SIZE", 13));
}
/*******************************************************************************
**
*******************************************************************************/

36
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModuleTest.java
View File

@ -42,15 +42,17 @@ import com.kingsrook.qqq.backend.core.state.SimpleStateKey;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.AUTH0_ACCESS_TOKEN_KEY;
import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.ACCESS_TOKEN_KEY;
import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.BASIC_AUTH_KEY;
import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.COULD_NOT_DECODE_ERROR;
import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.EXPIRED_TOKEN_ERROR;
import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.INVALID_TOKEN_ERROR;
import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.TOKEN_NOT_PROVIDED_ERROR;
import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.maskForLog;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
@ -143,7 +145,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
public void testInvalidToken()
{
Map<String, String> context = new HashMap<>();
context.put(AUTH0_ACCESS_TOKEN_KEY, INVALID_TOKEN);
context.put(ACCESS_TOKEN_KEY, INVALID_TOKEN);
try
{
@ -167,7 +169,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
public void testUndecodableToken()
{
Map<String, String> context = new HashMap<>();
context.put(AUTH0_ACCESS_TOKEN_KEY, UNDECODABLE_TOKEN);
context.put(ACCESS_TOKEN_KEY, UNDECODABLE_TOKEN);
try
{
@ -191,7 +193,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
public void testProperlyFormattedButExpiredToken()
{
Map<String, String> context = new HashMap<>();
context.put(AUTH0_ACCESS_TOKEN_KEY, EXPIRED_TOKEN);
context.put(ACCESS_TOKEN_KEY, EXPIRED_TOKEN);
try
{
@ -236,7 +238,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
public void testNullToken()
{
Map<String, String> context = new HashMap<>();
context.put(AUTH0_ACCESS_TOKEN_KEY, null);
context.put(ACCESS_TOKEN_KEY, null);
try
{
@ -267,7 +269,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
auth0Spy.createSession(qInstance, context);
auth0Spy.createSession(qInstance, context);
auth0Spy.createSession(qInstance, context);
verify(auth0Spy, times(1)).getAccessTokenFromAuth0(any(), any(), any());
verify(auth0Spy, times(1)).getAccessTokenForUsernameAndPasswordFromAuth0(any(), any(), any());
}
@ -467,4 +469,26 @@ public class Auth0AuthenticationModuleTest extends BaseTest
return (encoder.encodeToString(originalString.getBytes()));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testMask()
{
assertNull(maskForLog(null));
assertEquals("******", maskForLog("1"));
assertEquals("******", maskForLog("12"));
assertEquals("******", maskForLog("123"));
assertEquals("******", maskForLog("1234"));
assertEquals("******", maskForLog("12345"));
assertEquals("******", maskForLog("12345"));
assertEquals("******", maskForLog("123456"));
assertEquals("******", maskForLog("1234567"));
assertEquals("123456******", maskForLog("12345678"));
assertEquals("123456******", maskForLog("123456789"));
assertEquals("123456******", maskForLog("1234567890"));
}
}

39
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge;
import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
@ -45,6 +46,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@ -813,6 +815,11 @@ public class TestUtils
.withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of(youngPersonLimitDate))))
.withCodeReference(new QCodeReference(CheckAge.class))
)
.withAction(new TableAutomationAction()
.withName("failAutomationForSith")
.withTriggerEvent(TriggerEvent.POST_INSERT)
.withCodeReference(new QCodeReference(FailAutomationForSith.class))
)
.withAction(new TableAutomationAction()
.withName("increaseBirthdate")
.withTriggerEvent(TriggerEvent.POST_INSERT)
@ -918,6 +925,15 @@ public class TestUtils
List<QRecord> recordsToUpdate = new ArrayList<>();
for(QRecord record : recordAutomationInput.getRecordList())
{
////////////////////////////////////////////////////////////////////////
// get the record - its automation status should currently be RUNNING //
////////////////////////////////////////////////////////////////////////
QRecord freshlyFetchedRecord = new GetAction().executeForRecord(new GetInput(TABLE_NAME_PERSON_MEMORY).withPrimaryKey(record.getValue("id")));
assertEquals(AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), freshlyFetchedRecord.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()));
///////////////////////////////////////////
// do whatever business logic we do here //
///////////////////////////////////////////
LocalDate birthDate = record.getValueLocalDate("birthDate");
if(birthDate != null && birthDate.isAfter(limitDate))
{
@ -940,6 +956,29 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static class FailAutomationForSith extends RecordAutomationHandler
{
/*******************************************************************************
**
*******************************************************************************/
public void execute(RecordAutomationInput recordAutomationInput) throws QException
{
for(QRecord record : recordAutomationInput.getRecordList())
{
if("Darth".equals(record.getValue("firstName")))
{
throw new QException("Oops, you look like a Sith!");
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/

27
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java
View File

@ -1307,7 +1307,7 @@ public class BaseAPIActionUtil
*******************************************************************************/
protected void throwUnsupportedCriteriaField(QFilterCriteria criteria) throws QUserFacingException
{
throw new QUserFacingException("Unsupported query field [" + criteria.getFieldName() + "]");
throw new QUserFacingException("Unsupported query field: " + getFieldLabelFromCriteria(criteria));
}
@ -1317,7 +1317,30 @@ public class BaseAPIActionUtil
*******************************************************************************/
protected void throwUnsupportedCriteriaOperator(QFilterCriteria criteria) throws QUserFacingException
{
throw new QUserFacingException("Unsupported operator [" + criteria.getOperator() + "] for query field [" + criteria.getFieldName() + "]");
throw new QUserFacingException("Unsupported operator (" + criteria.getOperator() + ") for query field: " + getFieldLabelFromCriteria(criteria));
}
/*******************************************************************************
**
*******************************************************************************/
private String getFieldLabelFromCriteria(QFilterCriteria criteria)
{
String fieldLabel = criteria.getFieldName();
try
{
String label = actionInput.getTable().getField(criteria.getFieldName()).getLabel();
if(StringUtils.hasContent(label))
{
fieldLabel = label;
}
}
catch(Exception e)
{
LOG.debug("Error getting field label", e);
}
return fieldLabel;
}

23
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
View File

@ -863,6 +863,16 @@ public abstract class AbstractRDBMSAction implements QActionInterface
/*******************************************************************************
** Make it easy (e.g., for tests) to turn on poor-man's formatting of SQL
*******************************************************************************/
public static void setLogSQLReformat(boolean doReformat)
{
System.setProperty("qqq.rdbms.logSQL.reformat", String.valueOf(doReformat));
}
/*******************************************************************************
**
*******************************************************************************/
@ -874,6 +884,19 @@ public abstract class AbstractRDBMSAction implements QActionInterface
{
params = params.size() <= 100 ? params : params.subList(0, 99);
/////////////////////////////////////////////////////////////////////////////
// (very very) poor man's version of sql formatting... if property is true //
/////////////////////////////////////////////////////////////////////////////
if(System.getProperty("qqq.rdbms.logSQL.reformat", "false").equalsIgnoreCase("true"))
{
sql = Objects.requireNonNullElse(sql, "").toString()
.replaceAll("FROM ", "\nFROM\n ")
.replaceAll("INNER", "\n INNER")
.replaceAll("LEFT", "\n LEFT")
.replaceAll("RIGHT", "\n RIGHT")
.replaceAll("WHERE", "\nWHERE\n ");
}
if(System.getProperty("qqq.rdbms.logSQL.output", "logger").equalsIgnoreCase("system.out"))
{
System.out.println("SQL: " + sql);

1
qqq-dev-tools/bin/xbar-circleci-latest.sh
View File

@ -51,6 +51,7 @@ checkBuild()
qqq-frontend-material-dashboard) shortRepo="qfmd";;
ColdTrack-Live) shortRepo="ctl";;
ColdTrack-Live-Scripts) shortRepo="cls";;
Infoplus-Scripts) shortRepo="ips";;
esac
timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S%z" $(echo "$startDate" | sed 's/\....Z/+0000/') +%s)

27
qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/ExecuteCodeActionTest.java
View File

@ -30,6 +30,7 @@ import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.scripts.ExecuteCodeAction;
import com.kingsrook.qqq.backend.core.actions.scripts.QCodeExecutor;
import com.kingsrook.qqq.backend.core.actions.scripts.QCodeExecutorAware;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
@ -307,6 +308,32 @@ class ExecuteCodeActionTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testDeploymentModeIsInContext() throws QException
{
String scriptSource = """
return (deploymentMode);
""";
//////////////////////////////////////////////////////////////////////////////////////////////////
// first, with no deployment mode in the qInstance, make sure we can run, but get a null output //
//////////////////////////////////////////////////////////////////////////////////////////////////
OneTestOutput oneTestOutput = testOne(null, scriptSource, new HashMap<>());
assertNull(oneTestOutput.executeCodeOutput.getOutput());
/////////////////////////////////////////////////////////////////////
// next, set a deploymentMode, and assert that we get it back out. //
/////////////////////////////////////////////////////////////////////
QContext.getQInstance().setDeploymentMode("unit-test");
oneTestOutput = testOne(null, scriptSource, new HashMap<>());
assertEquals("unit-test", oneTestOutput.executeCodeOutput.getOutput());
}
/*******************************************************************************
**
*******************************************************************************/

105
qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java
View File

@ -27,7 +27,6 @@ import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
@ -58,7 +57,6 @@ import com.kingsrook.qqq.api.model.openapi.HttpMethod;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.AccessTokenException;
import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
@ -75,15 +73,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.Auth0AuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
@ -137,11 +132,6 @@ public class QJavalinApiHandler
{
return (() ->
{
/////////////////////////////
// authentication endpoint //
/////////////////////////////
ApiBuilder.post("/api/oauth/token", QJavalinApiHandler::handleAuthorization);
///////////////////////////////////////////////
// static endpoints to support rapidoc pages //
///////////////////////////////////////////////
@ -583,101 +573,6 @@ public class QJavalinApiHandler
/*******************************************************************************
**
*******************************************************************************/
private static void handleAuthorization(Context context)
{
try
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// clientId & clientSecret may either be provided as formParams, or in an Authorization: Basic header //
////////////////////////////////////////////////////////////////////////////////////////////////////////
String clientId;
String clientSecret;
String authorizationHeader = context.header("Authorization");
if(authorizationHeader != null && authorizationHeader.startsWith("Basic "))
{
try
{
byte[] credDecoded = Base64.getDecoder().decode(authorizationHeader.replace("Basic ", ""));
String credentials = new String(credDecoded, StandardCharsets.UTF_8);
String[] parts = credentials.split(":", 2);
clientId = parts[0];
clientSecret = parts[1];
}
catch(Exception e)
{
context.status(HttpStatus.BAD_REQUEST_400);
context.result("Could not parse client_id and client_secret from Basic Authorization header.");
return;
}
}
else
{
clientId = context.formParam("client_id");
if(clientId == null)
{
context.status(HttpStatus.BAD_REQUEST_400);
context.result("'client_id' must be provided.");
return;
}
clientSecret = context.formParam("client_secret");
if(clientSecret == null)
{
context.status(HttpStatus.BAD_REQUEST_400);
context.result("'client_secret' must be provided.");
return;
}
}
////////////////////////////////////////////////////////
// get the auth0 authentication module from qInstance //
////////////////////////////////////////////////////////
Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher();
QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(qInstance.getAuthentication());
try
{
//////////////////////////////////////////////////////////////////////////////////////////
// make call to get access token data, if no exception thrown, assume 200 OK and return //
//////////////////////////////////////////////////////////////////////////////////////////
QContext.init(qInstance, null); // hmm...
String accessToken = authenticationModule.createAccessToken(metaData, clientId, clientSecret);
context.status(HttpStatus.Code.OK.getCode());
context.result(accessToken);
QJavalinAccessLogger.logEndSuccess();
}
catch(AccessTokenException aae)
{
LOG.info("Error getting api access token", aae, logPair("clientId", clientId));
///////////////////////////////////////////////////////////////////////////
// if the exception has a status code, then return that code and message //
///////////////////////////////////////////////////////////////////////////
if(aae.getStatusCode() != null)
{
context.status(aae.getStatusCode());
context.result(aae.getMessage());
QJavalinAccessLogger.logEndSuccess();
}
////////////////////////////////////////////////////////
// if no code, throw and handle like other exceptions //
////////////////////////////////////////////////////////
throw (aae);
}
}
catch(Exception e)
{
handleException(context, e);
QJavalinAccessLogger.logEndFail(e);
}
}
/*******************************************************************************
**
*******************************************************************************/

51
qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java
View File

@ -51,7 +51,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.FullyAnonymousAuthenticationModule;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.javalin.QJavalinImplementation;
@ -1386,56 +1385,6 @@ class QJavalinApiHandlerTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAuthorizeNoParams()
{
///////////////
// no params //
///////////////
HttpResponse<String> response = Unirest.post(BASE_URL + "/api/oauth/token").asString();
assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus());
assertThat(response.getBody()).contains("client_id");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAuthorizeOneParam()
{
///////////////
// no params //
///////////////
HttpResponse<String> response = Unirest.post(BASE_URL + "/api/oauth/token")
.body("client_id=XXXXXXXXXX").asString();
assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus());
assertThat(response.getBody()).contains("client_secret");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAuthorizeAllParams()
{
///////////////
// no params //
///////////////
HttpResponse<String> response = Unirest.post(BASE_URL + "/api/oauth/token")
.body("client_id=XXXXXXXXXX&client_secret=YYYYYYYYYYYY").asString();
assertEquals(HttpStatus.OK_200, response.getStatus());
assertThat(response.getBody()).isEqualTo(FullyAnonymousAuthenticationModule.TEST_ACCESS_TOKEN);
}
/*******************************************************************************
**
*******************************************************************************/

73
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -113,6 +113,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.statusmessages.QStatusMessage;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
@ -148,10 +149,10 @@ public class QJavalinImplementation
{
private static final QLogger LOG = QLogger.getLogger(QJavalinImplementation.class);
public static final int SESSION_COOKIE_AGE = 60 * 60 * 24;
public static final String SESSION_ID_COOKIE_NAME = "sessionId";
public static final String BASIC_AUTH_NAME = "basicAuthString";
public static final String API_KEY_NAME = "apiKey";
public static final int SESSION_COOKIE_AGE = 60 * 60 * 24;
public static final String SESSION_ID_COOKIE_NAME = "sessionId";
public static final String SESSION_UUID_COOKIE_NAME = "sessionUUID";
public static final String API_KEY_NAME = "apiKey";
static QInstance qInstance;
static QJavalinMetaData javalinMetaData;
@ -159,8 +160,8 @@ public class QJavalinImplementation
private static Supplier<QInstance> qInstanceHotSwapSupplier;
private static long lastQInstanceHotSwapMillis;
private static final long MILLIS_BETWEEN_HOT_SWAPS = 2500;
public static final long SLOW_LOG_THRESHOLD_MS = 1000;
private static long MILLIS_BETWEEN_HOT_SWAPS = 2500;
public static final long SLOW_LOG_THRESHOLD_MS = 1000;
private static final Integer DEFAULT_COUNT_TIMEOUT_SECONDS = 60;
private static final Integer DEFAULT_QUERY_TIMEOUT_SECONDS = 60;
@ -329,6 +330,8 @@ public class QJavalinImplementation
{
return (() ->
{
post("/manageSession", QJavalinImplementation::manageSession);
/////////////////////
// metadata routes //
/////////////////////
@ -400,6 +403,36 @@ public class QJavalinImplementation
/*******************************************************************************
**
*******************************************************************************/
private static void manageSession(Context context)
{
try
{
Map<?, ?> map = context.bodyAsClass(Map.class);
QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher();
QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(qInstance.getAuthentication());
Map<String, String> authContext = new HashMap<>();
//? authContext.put("uuid", ValueUtils.getValueAsString(map.get("uuid")));
authContext.put(Auth0AuthenticationModule.ACCESS_TOKEN_KEY, ValueUtils.getValueAsString(map.get("accessToken")));
authContext.put(Auth0AuthenticationModule.DO_STORE_USER_SESSION_KEY, "true");
QSession session = authenticationModule.createSession(qInstance, authContext);
context.cookie(SESSION_UUID_COOKIE_NAME, session.getUuid(), SESSION_COOKIE_AGE);
context.result(JsonUtils.toJson(MapBuilder.of("uuid", session.getUuid())));
}
catch(Exception e)
{
handleException(context, e);
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -443,16 +476,24 @@ public class QJavalinImplementation
Map<String, String> authenticationContext = new HashMap<>();
String sessionIdCookieValue = context.cookie(SESSION_ID_COOKIE_NAME);
String sessionUuidCookieValue = context.cookie(Auth0AuthenticationModule.SESSION_UUID_KEY);
String authorizationHeaderValue = context.header("Authorization");
String apiKeyHeaderValue = context.header("x-api-key");
if(StringUtils.hasContent(sessionIdCookieValue))
{
////////////////////////////////////////
// first, look for a sessionId cookie //
////////////////////////////////////////
///////////////////////////////////////////////////////
// sessionId - maybe used by table-based auth module //
///////////////////////////////////////////////////////
authenticationContext.put(SESSION_ID_COOKIE_NAME, sessionIdCookieValue);
}
else if(StringUtils.hasContent(sessionUuidCookieValue))
{
///////////////////////////////////////////////////////////////////////////
// session UUID - known to be used by auth0 module (in aug. 2023 update) //
///////////////////////////////////////////////////////////////////////////
authenticationContext.put(Auth0AuthenticationModule.SESSION_UUID_KEY, sessionUuidCookieValue);
}
else if(apiKeyHeaderValue != null)
{
/////////////////////////////////////////////////////////////////
@ -533,12 +574,12 @@ public class QJavalinImplementation
if(authorizationHeaderValue.startsWith(basicPrefix))
{
authorizationHeaderValue = authorizationHeaderValue.replaceFirst(basicPrefix, "");
authenticationContext.put(BASIC_AUTH_NAME, authorizationHeaderValue);
authenticationContext.put(Auth0AuthenticationModule.BASIC_AUTH_KEY, authorizationHeaderValue);
}
else if(authorizationHeaderValue.startsWith(bearerPrefix))
{
authorizationHeaderValue = authorizationHeaderValue.replaceFirst(bearerPrefix, "");
authenticationContext.put(SESSION_ID_COOKIE_NAME, authorizationHeaderValue);
authenticationContext.put(Auth0AuthenticationModule.ACCESS_TOKEN_KEY, authorizationHeaderValue);
}
else
{
@ -1818,4 +1859,14 @@ public class QJavalinImplementation
return StringUtils.joinWithCommasAndAnd(errors.stream().map(QStatusMessage::getMessage).toList());
}
/*******************************************************************************
**
*******************************************************************************/
public static void setMillisBetweenHotSwaps(long millisBetweenHotSwaps)
{
MILLIS_BETWEEN_HOT_SWAPS = millisBetweenHotSwaps;
}
}

139
qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
View File

@ -29,9 +29,18 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import kong.unirest.HttpResponse;
import kong.unirest.Unirest;
import org.eclipse.jetty.http.HttpStatus;
@ -43,6 +52,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -635,7 +645,8 @@ class QJavalinImplementationTest extends QJavalinTestBase
JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
assertNotNull(jsonObject);
assertEquals(1, jsonObject.getInt("deletedRecordCount"));
TestUtils.runTestSql("SELECT id FROM person", (rs -> {
TestUtils.runTestSql("SELECT id FROM person", (rs ->
{
int rowsFound = 0;
while(rs.next())
{
@ -832,4 +843,130 @@ class QJavalinImplementationTest extends QJavalinTestBase
assertTrue(jsonObject.has("type"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testManageSession()
{
String body = """
{
"accessToken": "abcd",
"doStoreUserSession": true
}
""";
HttpResponse<String> response = Unirest.post(BASE_URL + "/manageSession")
.header("Content-Type", "application/json")
.body(body)
.asString();
assertEquals(200, response.getStatus());
JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
assertNotNull(jsonObject);
assertTrue(jsonObject.has("uuid"));
response.getHeaders().get("Set-Cookie").stream().anyMatch(s -> s.contains("sessionUUID"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testHotSwap() throws QInstanceValidationException
{
try
{
Function<String, QInstance> makeNewInstanceWithBackendName = (backendName) ->
{
QInstance newInstance = new QInstance();
newInstance.addBackend(new QBackendMetaData().withName(backendName).withBackendType("mock"));
if(!"invalid".equals(backendName))
{
newInstance.addTable(new QTableMetaData()
.withName("newTable")
.withBackendName(backendName)
.withField(new QFieldMetaData("newField", QFieldType.INTEGER))
.withPrimaryKeyField("newField")
);
}
return (newInstance);
};
QJavalinImplementation.setQInstanceHotSwapSupplier(() -> makeNewInstanceWithBackendName.apply("newBackend"));
/////////////////////////////////////////////////////////////////////////////////
// make sure before a hot-swap, that the instance doesn't have our new backend //
/////////////////////////////////////////////////////////////////////////////////
assertNull(QJavalinImplementation.qInstance.getBackend("newBackend"));
///////////////////////////////////////////////////////
// do a hot-swap, make sure the new backend is there //
///////////////////////////////////////////////////////
QJavalinImplementation.hotSwapQInstance(null);
assertNotNull(QJavalinImplementation.qInstance.getBackend("newBackend"));
////////////////////////////////////////////////////////////////////////////////////////////////////////////
// now change to make a different backend - try to swap again - but the newer backend shouldn't be there, //
// because the millis-between-hot-swaps won't have passed //
////////////////////////////////////////////////////////////////////////////////////////////////////////////
QJavalinImplementation.setQInstanceHotSwapSupplier(() -> makeNewInstanceWithBackendName.apply("newerBackend"));
QJavalinImplementation.hotSwapQInstance(null);
assertNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
////////////////////////////////////////////////////////////////////////////////////////////
// set the sleep threshold to 1 milli, sleep for 2, and then assert that we do swap again //
////////////////////////////////////////////////////////////////////////////////////////////
QJavalinImplementation.setMillisBetweenHotSwaps(1);
SleepUtils.sleep(2, TimeUnit.MILLISECONDS);
QJavalinImplementation.setQInstanceHotSwapSupplier(() -> makeNewInstanceWithBackendName.apply("newerBackend"));
QJavalinImplementation.hotSwapQInstance(null);
assertNotNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
////////////////////////////////////////////////////////////
// assert that an invalid instance doesn't get swapped in //
// e.g., "newerBackend" still exists //
////////////////////////////////////////////////////////////
SleepUtils.sleep(2, TimeUnit.MILLISECONDS);
QJavalinImplementation.setQInstanceHotSwapSupplier(() -> makeNewInstanceWithBackendName.apply("invalid"));
QJavalinImplementation.hotSwapQInstance(null);
assertNotNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
///////////////////////////////////////////////////////
// assert that if the supplier throws, we don't swap //
// e.g., "newerBackend" still exists //
///////////////////////////////////////////////////////
SleepUtils.sleep(2, TimeUnit.MILLISECONDS);
QJavalinImplementation.setQInstanceHotSwapSupplier(() ->
{
throw new RuntimeException("oops");
});
QJavalinImplementation.hotSwapQInstance(null);
assertNotNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
/////////////////////////////////////////////////////////////
// assert that if the supplier returns null, we don't swap //
// e.g., "newerBackend" still exists //
/////////////////////////////////////////////////////////////
SleepUtils.sleep(2, TimeUnit.MILLISECONDS);
QJavalinImplementation.setQInstanceHotSwapSupplier(() -> null);
QJavalinImplementation.hotSwapQInstance(null);
assertNotNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
}
finally
{
////////////////////////////////////////////////////////////
// restore things to how they used to be, for other tests //
////////////////////////////////////////////////////////////
QInstance qInstance = TestUtils.defineInstance();
QJavalinImplementation.setQInstanceHotSwapSupplier(null);
restartServerWithInstance(qInstance);
}
}
}

15
qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java
View File

@ -90,8 +90,6 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
import io.github.cdimascio.dotenv.Dotenv;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.core.config.Configurator;
import org.jline.reader.LineReader;
import org.jline.reader.LineReaderBuilder;
import org.jline.utils.Log;
import picocli.CommandLine;
import picocli.CommandLine.Model.CommandSpec;
@ -292,18 +290,7 @@ public class QPicoCliImplementation
}
Map<String, String> authenticationContext = new HashMap<>();
if(sessionId == null && authenticationModule instanceof Auth0AuthenticationModule)
{
LineReader lr = LineReaderBuilder.builder().build();
String tokenId = lr.readLine("Create a .env file with the contents of the Auth0 JWT Id Token in the variable 'SESSION_ID': \nPress enter once complete...");
dotenv = loadDotEnv();
if(dotenv.isPresent())
{
sessionId = dotenv.get().get("SESSION_ID");
}
}
authenticationContext.put("sessionId", sessionId);
authenticationContext.put(Auth0AuthenticationModule.ACCESS_TOKEN_KEY, sessionId);
// todo - does this need some per-provider logic actually? mmm...
session = authenticationModule.createSession(qInstance, authenticationContext);