QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-21 14:38:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged dev into feature/join-enhancements

Browse Source
This commit is contained in:
darin.kelkhoff
2023-09-27 09:16:08 -05:00
parent 831ac3bc07 71302eefdf
commit 623f3aca93
37 changed files with 2069 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java
View File

@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@ -49,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -108,12 +110,26 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
/*******************************************************************************
** Simple overload that internally figures out primary key and security key values
**
** Be aware - if the record doesn't have its security key values set (say it's a
** partial record as part of an update), then those values won't be in the
** security key map... This should probably be considered a bug.
*******************************************************************************/
public static void appendToInput(AuditInput auditInput, QTableMetaData table, QRecord record, String auditMessage)
{
appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), auditMessage);
}
/*******************************************************************************
** Add 1 auditSingleInput to an AuditInput object - with no details (child records).
*******************************************************************************/
public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
public static void appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
{
return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null));
appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null);
}
@ -139,6 +155,44 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
/*******************************************************************************
** For a given record, from a given table, build a map of the record's security
** key values.
**
** If, in case, the record has null value(s), and the oldRecord is given (e.g.,
** for the case of an update, where the record may not have all fields set, and
** oldRecord should be known for doing field-diffs), then try to get the value(s)
** from oldRecord.
**
** Currently, will leave values null if they aren't found after that.
**
** An alternative could be to re-fetch the record from its source if needed...
*******************************************************************************/
public static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional<QRecord> oldRecord)
{
Map<String, Serializable> securityKeyValues = new HashMap<>();
for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
{
Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
if(keyValue == null && oldRecord.isPresent())
{
LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
}
if(keyValue == null)
{
LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
}
securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
}
return securityKeyValues;
}
/*******************************************************************************
**
*******************************************************************************/

397
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java
View File

@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.actions.audits;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@ -52,13 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import static com.kingsrook.qqq.backend.core.actions.audits.AuditAction.getRecordSecurityKeyValues;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -70,6 +71,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
{
private static final QLogger LOG = QLogger.getLogger(DMLAuditAction.class);
public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
/*******************************************************************************
@ -99,34 +102,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
return (output);
}
String contextSuffix = "";
if(StringUtils.hasContent(input.getAuditContext()))
{
contextSuffix = " " + input.getAuditContext();
}
Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
{
String processName = runProcessInput.getProcessName();
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
if(process != null)
{
contextSuffix = " during process: " + process.getLabel();
}
}
QSession qSession = QContext.getQSession();
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
{
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
{
apiLabel = "API";
}
contextSuffix += (" via " + apiLabel + " Version: " + apiVersion);
}
String contextSuffix = getContentSuffix(input);
AuditInput auditInput = new AuditInput();
if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields))
@ -137,7 +113,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : recordList)
{
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix);
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), "Record was " + dmlType.pastTenseVerb + contextSuffix);
}
}
else if(auditLevel.equals(AuditLevel.FIELD))
@ -147,7 +123,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
///////////////////////////////////////////////////////////////////
// do many audits, all with field level details, for FIELD level //
///////////////////////////////////////////////////////////////////
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), qSession);
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
qPossibleValueTranslator.translatePossibleValuesInRecords(table, CollectionUtils.mergeLists(recordList, oldRecordList));
//////////////////////////////////////////
@ -169,92 +145,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
List<QRecord> details = new ArrayList<>();
for(String fieldName : sortedFieldNames)
{
if(!record.getValues().containsKey(fieldName))
{
////////////////////////////////////////////////////////////////////////////////////////////////
// if the stored record doesn't have this field name, then don't audit anything about it //
// this is to deal with our Patch style updates not looking like every field was cleared out. //
////////////////////////////////////////////////////////////////////////////////////////////////
continue;
}
if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
{
continue;
}
QFieldMetaData field = table.getField(fieldName);
Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
QRecord detailRecord = null;
if(oldRecord == null)
{
if(DMLType.INSERT.equals(dmlType) && value == null)
{
continue;
}
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
detailRecord.withValue("newValue", formattedValue);
}
}
else
{
if(!Objects.equals(oldValue, value))
{
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
}
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("newValue", formattedValue);
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
detailRecord.withValue("oldValue", formattedOldValue);
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("oldValue", formattedOldValue);
detailRecord.withValue("newValue", formattedValue);
}
}
}
}
if(detailRecord != null)
{
detailRecord.withValue("fieldName", fieldName);
details.add(detailRecord);
}
makeAuditDetailRecordForField(fieldName, table, dmlType, record, oldRecord)
.ifPresent(details::add);
}
if(details.isEmpty() && DMLType.UPDATE.equals(dmlType))
@ -264,7 +156,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
}
else
{
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.ofNullable(oldRecord)), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
}
}
}
@ -284,6 +176,256 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
static String getContentSuffix(DMLAuditInput input)
{
StringBuilder contextSuffix = new StringBuilder();
/////////////////////////////////////////////////////////////////////////////
// start with context from the input wrapper //
// note, these contexts get propagated down from Input/Update/Delete Input //
/////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(input.getAuditContext()))
{
contextSuffix.append(" ").append(input.getAuditContext());
}
/////////////////////////////////////////////////////////////////////////////////////
// note process label (and a possible context from the process's state) if present //
/////////////////////////////////////////////////////////////////////////////////////
Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
{
String processAuditContext = ValueUtils.getValueAsString(runProcessInput.getValue(AUDIT_CONTEXT_FIELD_NAME));
if(StringUtils.hasContent(processAuditContext))
{
contextSuffix.append(" ").append(processAuditContext);
}
String processName = runProcessInput.getProcessName();
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
if(process != null)
{
contextSuffix.append(" during process: ").append(process.getLabel());
}
}
///////////////////////////////////////////////////
// use api label & version if present in session //
///////////////////////////////////////////////////
QSession qSession = QContext.getQSession();
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
{
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
{
apiLabel = "API";
}
contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
}
return (contextSuffix.toString());
}
/*******************************************************************************
**
*******************************************************************************/
static Optional<QRecord> makeAuditDetailRecordForField(String fieldName, QTableMetaData table, DMLType dmlType, QRecord record, QRecord oldRecord)
{
if(!record.getValues().containsKey(fieldName))
{
////////////////////////////////////////////////////////////////////////////////////////////////
// if the stored record doesn't have this field name, then don't audit anything about it //
// this is to deal with our Patch style updates not looking like every field was cleared out. //
////////////////////////////////////////////////////////////////////////////////////////////////
return (Optional.empty());
}
if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
{
return (Optional.empty());
}
QFieldMetaData field = table.getField(fieldName);
Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
QRecord detailRecord = null;
if(oldRecord == null)
{
if(DMLType.INSERT.equals(dmlType) && value == null)
{
return (Optional.empty());
}
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
detailRecord.withValue("newValue", formattedValue);
}
}
else
{
if(areValuesDifferentForAudit(field, value, oldValue))
{
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
}
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("newValue", formattedValue);
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
detailRecord.withValue("oldValue", formattedOldValue);
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("oldValue", formattedOldValue);
detailRecord.withValue("newValue", formattedValue);
}
}
}
}
if(detailRecord != null)
{
////////////////////////////////////////////////////////////////////
// useful if doing dev in here - but overkill for any other time. //
////////////////////////////////////////////////////////////////////
// LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
detailRecord.withValue("fieldName", fieldName);
return (Optional.of(detailRecord));
}
return (Optional.empty());
}
/*******************************************************************************
**
*******************************************************************************/
static boolean areValuesDifferentForAudit(QFieldMetaData field, Serializable value, Serializable oldValue)
{
try
{
///////////////////
// decimal rules //
///////////////////
if(field.getType().equals(QFieldType.DECIMAL))
{
BigDecimal newBD = ValueUtils.getValueAsBigDecimal(value);
BigDecimal oldBD = ValueUtils.getValueAsBigDecimal(oldValue);
if(newBD == null && oldBD == null)
{
return (false);
}
if(newBD == null || oldBD == null)
{
return (true);
}
return (newBD.compareTo(oldBD) != 0);
}
////////////////////
// dateTime rules //
////////////////////
if(field.getType().equals(QFieldType.DATE_TIME))
{
Instant newI = ValueUtils.getValueAsInstant(value);
Instant oldI = ValueUtils.getValueAsInstant(oldValue);
if(newI == null && oldI == null)
{
return (false);
}
if(newI == null || oldI == null)
{
return (true);
}
////////////////////////////////
// just compare to the second //
////////////////////////////////
return (newI.truncatedTo(ChronoUnit.SECONDS).compareTo(oldI.truncatedTo(ChronoUnit.SECONDS)) != 0);
}
//////////////////
// string rules //
//////////////////
if(field.getType().isStringLike())
{
String newString = ValueUtils.getValueAsString(value);
String oldString = ValueUtils.getValueAsString(oldValue);
boolean newIsNullOrEmpty = !StringUtils.hasContent(newString);
boolean oldIsNullOrEmpty = !StringUtils.hasContent(oldString);
if(newIsNullOrEmpty && oldIsNullOrEmpty)
{
return (false);
}
if(newIsNullOrEmpty || oldIsNullOrEmpty)
{
return (true);
}
return (newString.compareTo(oldString) != 0);
}
/////////////////////////////////////
// default just use Objects.equals //
/////////////////////////////////////
return !Objects.equals(oldValue, value);
}
catch(Exception e)
{
LOG.debug("Error checking areValuesDifferentForAudit", e, logPair("fieldName", field.getName()), logPair("value", value), logPair("oldValue", oldValue));
}
////////////////////////////////////
// default to something simple... //
////////////////////////////////////
return !Objects.equals(oldValue, value);
}
/*******************************************************************************
**
*******************************************************************************/
@ -373,21 +515,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
{
Map<String, Serializable> securityKeyValues = new HashMap<>();
for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
{
securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
}
return securityKeyValues;
}
/*******************************************************************************
**
*******************************************************************************/
@ -407,7 +534,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
private enum DMLType
enum DMLType
{
INSERT("Inserted", true),
UPDATE("Edited", true),

37
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java
View File

@ -22,9 +22,8 @@
package com.kingsrook.qqq.backend.core.actions.automation;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
@ -90,6 +89,10 @@ public class RecordAutomationStatusUpdater
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Avoid setting records to PENDING_INSERT or PENDING_UPDATE even if they don't have any insert or update automations or triggers //
// such records should go straight to OK status. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(canWeSkipPendingAndGoToOkay(table, automationStatus))
{
automationStatus = AutomationStatus.OK;
@ -121,9 +124,13 @@ public class RecordAutomationStatusUpdater
** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just
** move the status straight to OK.
*******************************************************************************/
private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
{
List<TableAutomationAction> tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>());
List<TableAutomationAction> tableActions = Collections.emptyList();
if(table.getAutomationDetails() != null && table.getAutomationDetails().getActions() != null)
{
tableActions = table.getAutomationDetails().getActions();
}
if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
{
@ -135,6 +142,12 @@ public class RecordAutomationStatusUpdater
{
return (false);
}
////////////////////////////////////////////////////////////////////////////////////////
// if we're going to pending-insert, and there are no insert automations or triggers, //
// then we may skip pending and go to okay. //
////////////////////////////////////////////////////////////////////////////////////////
return (true);
}
else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
{
@ -146,9 +159,21 @@ public class RecordAutomationStatusUpdater
{
return (false);
}
}
return (true);
////////////////////////////////////////////////////////////////////////////////////////
// if we're going to pending-update, and there are no insert automations or triggers, //
// then we may skip pending and go to okay. //
////////////////////////////////////////////////////////////////////////////////////////
return (true);
}
else
{
///////////////////////////////////////////////////////////////////////////////////////////////////////
// if we're going to any other automation status - then we may never "skip pending" and go to okay - //
// because we weren't asked to go to pending! //
///////////////////////////////////////////////////////////////////////////////////////////////////////
return (false);
}
}

48
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java
View File

@ -342,22 +342,9 @@ public class PollingAutomationPerTableRunner implements Runnable
boolean anyActionsFailed = false;
for(TableAutomationAction action : actions)
{
try
boolean hadError = applyActionToRecords(table, records, action);
if(hadError)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
if(CollectionUtils.nullSafeHasContents(matchingQRecords))
{
LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
applyActionToMatchingRecords(table, matchingQRecords, action);
}
}
catch(Exception e)
{
LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
anyActionsFailed = true;
}
}
@ -377,6 +364,37 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
** Run one action over a list of records (if they match the action's filter).
**
** @return hadError - true if an exception was caught; false if all OK.
*******************************************************************************/
protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
{
try
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
if(CollectionUtils.nullSafeHasContents(matchingQRecords))
{
LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
applyActionToMatchingRecords(table, matchingQRecords, action);
}
return (false);
}
catch(Exception e)
{
LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
return (true);
}
}
/*******************************************************************************
** For a given action, and a list of records - return a new list, of the ones
** which match the action's filter (if there is one - if not, then all match).

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java
View File

@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLogg
import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -112,6 +114,11 @@ public class ExecuteCodeAction
context.putAll(input.getInput());
}
//////////////////////////////////////////
// safely always set the deploymentMode //
//////////////////////////////////////////
context.put("deploymentMode", ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getDeploymentMode(), null));
/////////////////////////////////////////////////////////////////////////////////
// set the qCodeExecutor into any context objects which are QCodeExecutorAware //
/////////////////////////////////////////////////////////////////////////////////

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java
View File

@ -244,8 +244,6 @@ public class SearchPossibleValueSourceAction
}
}
queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
// todo - skip & limit as params
queryFilter.setLimit(250);
@ -257,6 +255,9 @@ public class SearchPossibleValueSourceAction
input.getDefaultQueryFilter().addSubFilter(queryFilter);
queryFilter = input.getDefaultQueryFilter();
}
queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
queryInput.setFilter(queryFilter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java
View File

@ -84,7 +84,7 @@ public class QContext
actionStackThreadLocal.get().add(actionInput);
}
if(!qInstance.getHasBeenValidated())
if(qInstance != null && !qInstance.getHasBeenValidated())
{
try
{

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java
View File

@ -347,7 +347,7 @@ public class QMetaDataVariableInterpreter
if(canParseAsInteger(envValue))
{
LOG.info("Read env var [" + environmentVariableName + "] as integer " + environmentVariableName);
return (Integer.parseInt(propertyValue));
return (Integer.parseInt(envValue));
}
else
{

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/audits/AuditsMetaDataProvider.java
View File

@ -116,16 +116,19 @@ public class AuditsMetaDataProvider
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(TABLE_NAME_AUDIT_TABLE)
.withTableName(TABLE_NAME_AUDIT_TABLE)
.withOrderByField("name")
);
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(TABLE_NAME_AUDIT_USER)
.withTableName(TABLE_NAME_AUDIT_USER)
.withOrderByField("name")
);
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(TABLE_NAME_AUDIT)
.withTableName(TABLE_NAME_AUDIT)
.withOrderByField("id", false)
);
}

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java
View File

@ -93,6 +93,7 @@ public class QInstance
private Map<String, QSupplementalInstanceMetaData> supplementalMetaData = new LinkedHashMap<>();
private String deploymentMode;
private Map<String, String> environmentValues = new LinkedHashMap<>();
private String defaultTimeZoneId = "UTC";
@ -1165,4 +1166,36 @@ public class QInstance
}
this.joinGraph = joinGraph;
}
/*******************************************************************************
** Getter for deploymentMode
*******************************************************************************/
public String getDeploymentMode()
{
return (this.deploymentMode);
}
/*******************************************************************************
** Setter for deploymentMode
*******************************************************************************/
public void setDeploymentMode(String deploymentMode)
{
this.deploymentMode = deploymentMode;
}
/*******************************************************************************
** Fluent setter for deploymentMode
*******************************************************************************/
public QInstance withDeploymentMode(String deploymentMode)
{
this.deploymentMode = deploymentMode;
return (this);
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/authentication/Auth0AuthenticationMetaData.java
View File

@ -60,7 +60,6 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
private String auth0ClientSecretField;
private Serializable qqqRecordIdField;
/////////////////////////////////////
// fields on the accessToken table //
/////////////////////////////////////

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java
View File

@ -187,7 +187,8 @@ public class QueryStatMetaDataProvider
return (new QPossibleValueSource()
.withType(QPossibleValueSourceType.TABLE)
.withName(QueryStat.TABLE_NAME)
.withTableName(QueryStat.TABLE_NAME));
.withTableName(QueryStat.TABLE_NAME))
.withOrderByField("id", false);
}
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedfilters/SavedFiltersMetaDataProvider.java
View File

@ -88,7 +88,8 @@ public class SavedFiltersMetaDataProvider
.withName(SavedFilter.TABLE_NAME)
.withType(QPossibleValueSourceType.TABLE)
.withTableName(SavedFilter.TABLE_NAME)
.withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);
.withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
.withOrderByField("label");
}
}

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java
View File

@ -303,19 +303,24 @@ public class ScriptsMetaDataProvider
{
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(Script.TABLE_NAME)
.withTableName(Script.TABLE_NAME));
.withTableName(Script.TABLE_NAME)
.withOrderByField("name"));
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(ScriptRevision.TABLE_NAME)
.withTableName(ScriptRevision.TABLE_NAME));
.withTableName(ScriptRevision.TABLE_NAME)
.withOrderByField("scriptId")
.withOrderByField("sequenceNo", false));
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(ScriptType.TABLE_NAME)
.withTableName(ScriptType.TABLE_NAME));
.withTableName(ScriptType.TABLE_NAME)
.withOrderByField("name"));
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(ScriptLog.TABLE_NAME)
.withTableName(ScriptLog.TABLE_NAME));
.withTableName(ScriptLog.TABLE_NAME)
.withOrderByField("id", false));
instance.addPossibleValueSource(new QPossibleValueSource()
.withName(ScriptTypeFileMode.NAME)
@ -378,7 +383,7 @@ public class ScriptsMetaDataProvider
/*******************************************************************************
**
*******************************************************************************/
private QTableMetaData defineTableTriggerTable(String backendName) throws QException
public QTableMetaData defineTableTriggerTable(String backendName) throws QException
{
QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
.withRecordLabelFields("id")

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java
View File

@ -127,7 +127,8 @@ public class QQQTablesMetaDataProvider
return (new QPossibleValueSource()
.withType(QPossibleValueSourceType.TABLE)
.withName(QQQTable.TABLE_NAME)
.withTableName(QQQTable.TABLE_NAME));
.withTableName(QQQTable.TABLE_NAME))
.withOrderByField("label");
}
}

439
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java
View File

@ -33,7 +33,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import com.auth0.client.auth.AuthAPI;
import com.auth0.exception.Auth0Exception;
import com.auth0.json.auth.TokenHolder;
@ -52,6 +51,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.CapturedContext;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.AccessTokenException;
import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
@ -68,11 +68,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.Auth0AuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.model.UserSession;
import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.SimpleStateKey;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
@ -80,7 +80,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.http.HttpStatus;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
@ -90,9 +89,23 @@ import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** QQQ AuthenticationModule for working with Auth0.
**
** createSession can be called with the following fields in its context:
**
** System-User session use-case:
** 1: Takes in an "accessToken" (but doesn't store a userSession record).
** 1b: legacy frontend use-case does the same as system-user!
**
** Web User session use-cases:
** 2: creates a new session (userSession record) by taking an "accessToken"
** 3: looks up an existing session (userSession record) by taking a "sessionUUID"
** 4: takes an "apiKey" (looked up in metaData.AccessTokenTableName - refreshing accessToken with auth0 if needed).
** 5: takes a "basicAuthString" (encoded username:password), which make a new accessToken in auth0
**
*******************************************************************************/
public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
@ -104,14 +117,17 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
public static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 1800;
public static final String AUTH0_ACCESS_TOKEN_KEY = "sessionId";
public static final String API_KEY = "apiKey";
public static final String BASIC_AUTH_KEY = "basicAuthString";
public static final String ACCESS_TOKEN_KEY = "accessToken";
public static final String API_KEY = "apiKey"; // todo - look for users of this, see if we can change to use this constant; maybe move constants up?
public static final String SESSION_UUID_KEY = "sessionUUID";
public static final String BASIC_AUTH_KEY = "basicAuthString"; // todo - look for users of this, see if we can change to use this constant; maybe move constants up?
public static final String TOKEN_NOT_PROVIDED_ERROR = "Access Token was not provided";
public static final String COULD_NOT_DECODE_ERROR = "Unable to decode access token";
public static final String EXPIRED_TOKEN_ERROR = "Token has expired";
public static final String INVALID_TOKEN_ERROR = "An invalid token was provided";
public static final String DO_STORE_USER_SESSION_KEY = "doStoreUserSession";
static final String TOKEN_NOT_PROVIDED_ERROR = "Access Token was not provided";
static final String COULD_NOT_DECODE_ERROR = "Unable to decode access token";
static final String EXPIRED_TOKEN_ERROR = "Token has expired";
static final String INVALID_TOKEN_ERROR = "An invalid token was provided";
////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -149,94 +165,121 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
@Override
public QSession createSession(QInstance qInstance, Map<String, String> context) throws QAuthenticationException
{
Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
///////////////////////////////////////////////////////////
// check if we are processing a Basic Auth Session first //
///////////////////////////////////////////////////////////
if(context.containsKey(BASIC_AUTH_KEY))
{
AuthAPI auth = AuthAPI.newBuilder(metaData.getBaseUrl(), metaData.getClientId(), metaData.getClientSecret()).build();
try
{
/////////////////////////////////////////////////
// decode the credentials from the header auth //
/////////////////////////////////////////////////
String base64Credentials = context.get(BASIC_AUTH_KEY).trim();
String accessToken = getAccessTokenFromBase64BasicAuthCredentials(metaData, auth, base64Credentials);
context.put(AUTH0_ACCESS_TOKEN_KEY, accessToken);
}
catch(Auth0Exception e)
{
////////////////
// ¯\_(ツ)_/¯ //
////////////////
String message = "Error handling basic authentication: " + e.getMessage();
LOG.error(message, e);
throw (new QAuthenticationException(message));
}
}
////////////////////////////////////////////////////////////////////
// get the jwt id or qqq translated token from the context object //
////////////////////////////////////////////////////////////////////
String accessToken = context.get(AUTH0_ACCESS_TOKEN_KEY);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// check to see if the session id is a UUID, if so, that means we need to look up the 'actual' token in the access_token table //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(accessToken != null && StringUtils.isUUID(accessToken))
{
accessToken = lookupActualAccessToken(metaData, accessToken);
}
////////////////////////////////////////////////////////
// if access token is still null, look for an api key //
////////////////////////////////////////////////////////
if(accessToken == null)
{
String apiKey = context.get(API_KEY);
if(apiKey != null)
{
accessToken = getAccessTokenFromApiKey(metaData, apiKey);
}
}
if(accessToken == null)
{
LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
}
//////////////////////////////////////////////////////////////////////////////////////
// decode the token locally to make sure it is valid and to look at when it expires //
//////////////////////////////////////////////////////////////////////////////////////
try
{
Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
String accessToken = null;
if(CollectionUtils.containsKeyWithNonNullValue(context, SESSION_UUID_KEY))
{
/////////////////////////////////////////////////////////////////////////////////////////
// process a sessionUUID - looks up userSession record - cannot create token this way. //
/////////////////////////////////////////////////////////////////////////////////////////
String sessionUUID = context.get(SESSION_UUID_KEY);
LOG.info("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID)));
if(sessionUUID != null)
{
accessToken = getAccessTokenFromSessionUUID(metaData, sessionUUID);
}
}
else if(CollectionUtils.containsKeyWithNonNullValue(context, ACCESS_TOKEN_KEY))
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the context contains an access token, then create a new session based on that token. //
// todo#authHeader - this else/if should maybe be first, but while we have frontend passing both, we want it second //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
accessToken = context.get(ACCESS_TOKEN_KEY);
QSession qSession = buildAndValidateSession(qInstance, accessToken);
////////////////////////////////////////////////////////////////
// build & store userSession db record, if requested to do so //
////////////////////////////////////////////////////////////////
if(CollectionUtils.containsKeyWithNonNullValue(context, DO_STORE_USER_SESSION_KEY))
{
insertUserSession(qInstance, accessToken, qSession);
LOG.info("Creating session based on input accessToken and creating a userSession", logPair("userId", qSession.getUser().getIdReference()));
}
else
{
///////////////////////////////////////////////
// todo#authHeader - remove all this logging //
///////////////////////////////////////////////
String userName = qSession.getUser() != null ? qSession.getUser().getFullName() : null;
if(userName != null && !userName.contains("System User"))
{
LOG.info("Creating session based on input accessToken but not creating a userSession", logPair("userName", qSession.getUser().getFullName()));
}
}
return (qSession);
}
else if(CollectionUtils.containsKeyWithNonNullValue(context, BASIC_AUTH_KEY))
{
//////////////////////////////////////////////////////////////////////////////////////
// Process a basic auth (username:password) //
// by getting an access token from auth0 (re-using from state provider if possible) //
//////////////////////////////////////////////////////////////////////////////////////
AuthAPI auth = AuthAPI.newBuilder(metaData.getBaseUrl(), metaData.getClientId(), metaData.getClientSecret()).build();
try
{
/////////////////////////////////////////////////
// decode the credentials from the header auth //
/////////////////////////////////////////////////
String base64Credentials = context.get(BASIC_AUTH_KEY).trim();
LOG.info("Creating session from basicAuthentication", logPair("base64Credentials", maskForLog(base64Credentials)));
accessToken = getAccessTokenFromBase64BasicAuthCredentials(metaData, auth, base64Credentials);
}
catch(Auth0Exception e)
{
////////////////
// ¯\_(ツ)_/¯ //
////////////////
String message = "Error handling basic authentication: " + e.getMessage();
LOG.error(message, e);
throw (new QAuthenticationException(message));
}
}
else if(CollectionUtils.containsKeyWithNonNullValue(context, API_KEY))
{
///////////////////////////////////////////////////////////////////////////////////////
// process an api key - looks up client application token (creating token if needed) //
///////////////////////////////////////////////////////////////////////////////////////
String apiKey = context.get(API_KEY);
LOG.info("Creating session from apiKey (accessTokenTable)", logPair("apiKey", maskForLog(apiKey)));
if(apiKey != null)
{
accessToken = getAccessTokenFromApiKey(metaData, apiKey);
}
}
/* todo confirm this is deprecated
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// check to see if the session id is a UUID, if so, that means we need to look up the 'actual' token in the access_token table //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(accessToken != null && StringUtils.isUUID(accessToken))
{
accessToken = lookupActualAccessToken(metaData, accessToken);
}
*/
///////////////////////////////////////////
// if token wasn't found by now, give up //
///////////////////////////////////////////
if(accessToken == null)
{
LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
}
/////////////////////////////////////////////////////
// try to build session to see if still valid //
// then call method to check more session validity //
/////////////////////////////////////////////////////
QSession qSession = buildQSessionFromToken(accessToken, qInstance);
if(isSessionValid(qInstance, qSession))
{
return (qSession);
}
///////////////////////////////////////////////////////////////////////////////////////
// if we make it here it means we have never validated this token or its been a long //
// enough duration so we need to re-verify the token //
///////////////////////////////////////////////////////////////////////////////////////
qSession = revalidateTokenAndBuildSession(qInstance, accessToken);
////////////////////////////////////////////////////////////////////
// put now into state so we dont check until next interval passes //
///////////////////////////////////////////////////////////////////
StateProviderInterface spi = getStateProvider();
SimpleStateKey<String> key = new SimpleStateKey<>(qSession.getIdReference());
spi.put(key, Instant.now());
return (qSession);
return buildAndValidateSession(qInstance, accessToken);
}
catch(QAuthenticationException qae)
{
throw (qae);
}
catch(JWTDecodeException jde)
{
@ -272,6 +315,61 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
** Insert a session as a new record into userSession table
*******************************************************************************/
private void insertUserSession(QInstance qInstance, String accessToken, QSession qSession) throws QException
{
CapturedContext capturedContext = QContext.capture();
try
{
QContext.init(qInstance, null);
QContext.setQSession(getChickenAndEggSession());
UserSession userSession = new UserSession()
.withUuid(qSession.getUuid())
.withUserId(qSession.getUser().getIdReference())
.withAccessToken(accessToken);
new InsertAction().execute(new InsertInput(UserSession.TABLE_NAME).withRecordEntity(userSession));
}
finally
{
QContext.init(capturedContext);
}
}
/*******************************************************************************
**
*******************************************************************************/
private QSession buildAndValidateSession(QInstance qInstance, String accessToken) throws JwkException
{
QSession qSession = buildQSessionFromToken(accessToken, qInstance);
if(isSessionValid(qInstance, qSession))
{
return (qSession);
}
//////////////////////////////////////////////////////////////////////////////////////////
// if we make it here it means we have never validated this token or it has been a long //
// enough duration so we need to re-verify the token //
//////////////////////////////////////////////////////////////////////////////////////////
qSession = revalidateTokenAndBuildSession(qInstance, accessToken);
/////////////////////////////////////////////////////////////////////
// put now into state so we don't check until next interval passes //
/////////////////////////////////////////////////////////////////////
StateProviderInterface spi = getStateProvider();
SimpleStateKey<String> key = new SimpleStateKey<>(qSession.getIdReference());
spi.put(key, Instant.now());
return (qSession);
}
/*******************************************************************************
**
*******************************************************************************/
@ -299,7 +397,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
byte[] credDecoded = Base64.getDecoder().decode(base64Credentials);
String credentials = new String(credDecoded, StandardCharsets.UTF_8);
String accessToken = getAccessTokenFromAuth0(metaData, auth, credentials);
String accessToken = getAccessTokenForUsernameAndPasswordFromAuth0(metaData, auth, credentials);
stateProvider.put(accessTokenStateKey, accessToken);
stateProvider.put(timestampStateKey, Instant.now());
return (accessToken);
@ -310,7 +408,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
**
*******************************************************************************/
protected String getAccessTokenFromAuth0(Auth0AuthenticationMetaData metaData, AuthAPI auth, String credentials) throws Auth0Exception
protected String getAccessTokenForUsernameAndPasswordFromAuth0(Auth0AuthenticationMetaData metaData, AuthAPI auth, String credentials) throws Auth0Exception
{
/////////////////////////////////////
// call auth0 with a login request //
@ -620,75 +718,11 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
** create a new auth0 access token
** make http request to Auth0 for a new access token for an application - e.g.,
** with a clientId and clientSecret as params
**
*******************************************************************************/
public String createAccessToken(QAuthenticationMetaData metaData, String clientId, String clientSecret) throws AccessTokenException
{
QSession sessionBefore = QContext.getQSession();
Auth0AuthenticationMetaData auth0MetaData = (Auth0AuthenticationMetaData) metaData;
try
{
QContext.setQSession(getChickenAndEggSession());
///////////////////////////////////////////////////////////////////////////////////////
// fetch the application from database, will throw accesstokenexception if not found //
///////////////////////////////////////////////////////////////////////////////////////
QRecord clientAuth0Application = getClientAuth0Application(auth0MetaData, clientId);
/////////////////////////////////////////////////////////////////////////////////////////////////
// request access token from auth0 if exception is not thrown, that means 200OK, we want to //
// store the actual access token in the database, and return a unique value //
// back to the user which will be what they use on subseqeunt requests (because token too big) //
/////////////////////////////////////////////////////////////////////////////////////////////////
JSONObject accessTokenData = requestAccessTokenFromAuth0(auth0MetaData, clientId, clientSecret);
Integer expiresInSeconds = accessTokenData.getInt("expires_in");
String accessToken = accessTokenData.getString("access_token");
String uuid = UUID.randomUUID().toString();
/////////////////////////////////
// store the details in the db //
/////////////////////////////////
QRecord accessTokenRecord = new QRecord()
.withValue(auth0MetaData.getClientAuth0ApplicationIdField(), clientAuth0Application.getValue("id"))
.withValue(auth0MetaData.getAuth0AccessTokenField(), accessToken)
.withValue(auth0MetaData.getQqqAccessTokenField(), uuid)
.withValue(auth0MetaData.getExpiresInSecondsField(), expiresInSeconds);
InsertInput input = new InsertInput();
input.setTableName(auth0MetaData.getAccessTokenTableName());
input.setRecords(List.of(accessTokenRecord));
new InsertAction().execute(input);
//////////////////////////////////
// update and send the response //
//////////////////////////////////
accessTokenData.put("access_token", uuid);
accessTokenData.remove("scope");
return (accessTokenData.toString());
}
catch(AccessTokenException ate)
{
throw (ate);
}
catch(Exception e)
{
throw (new AccessTokenException(e.getMessage(), e));
}
finally
{
QContext.setQSession(sessionBefore);
}
}
/*******************************************************************************
** make http request to Auth0 for a new access token
**
*******************************************************************************/
public JSONObject requestAccessTokenFromAuth0(Auth0AuthenticationMetaData auth0MetaData, String clientId, String clientSecret) throws AccessTokenException
public JSONObject requestAccessTokenForClientIdAndSecretFromAuth0(Auth0AuthenticationMetaData auth0MetaData, String clientId, String clientSecret) throws AccessTokenException
{
///////////////////////////////////////////////////////////////////
// make a request to Auth0 using the client_id and client_secret //
@ -776,6 +810,63 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
** Look up access_token from session UUID
**
*******************************************************************************/
private String getAccessTokenFromSessionUUID(Auth0AuthenticationMetaData metaData, String sessionUUID) throws QAuthenticationException
{
String accessToken = null;
QSession beforeSession = QContext.getQSession();
try
{
QContext.setQSession(getChickenAndEggSession());
///////////////////////////////////////
// query for the user session record //
///////////////////////////////////////
QRecord userSessionRecord = new GetAction().executeForRecord(new GetInput(UserSession.TABLE_NAME)
.withUniqueKey(Map.of("uuid", sessionUUID))
.withShouldMaskPasswords(false)
.withShouldOmitHiddenFields(false));
if(userSessionRecord != null)
{
accessToken = userSessionRecord.getValueString("accessToken");
////////////////////////////////////////////////////////////
// decode the accessToken and make sure it is not expired //
////////////////////////////////////////////////////////////
if(accessToken != null)
{
DecodedJWT jwt = JWT.decode(accessToken);
if(jwt.getExpiresAtAsInstant().isBefore(Instant.now()))
{
throw (new QAuthenticationException("accessToken is expired"));
}
}
}
}
catch(QAuthenticationException qae)
{
throw (qae);
}
catch(Exception e)
{
LOG.warn("Error looking up userSession by sessionUUID", e);
throw (new QAuthenticationException("Error looking up userSession by sessionUUID", e));
}
finally
{
QContext.setQSession(beforeSession);
}
return (accessToken);
}
/*******************************************************************************
** Look up access_token from api key
**
@ -841,7 +932,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// store the actual access token in the database, and return a unique value //
// back to the user which will be what they use on subsequent requests (because token too big) //
/////////////////////////////////////////////////////////////////////////////////////////////////
JSONObject accessTokenData = requestAccessTokenFromAuth0(metaData, clientId, clientSecret);
JSONObject accessTokenData = requestAccessTokenForClientIdAndSecretFromAuth0(metaData, clientId, clientSecret);
Integer expiresInSeconds = accessTokenData.getInt("expires_in");
accessToken = accessTokenData.getString("access_token");
@ -872,25 +963,23 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/*******************************************************************************
** Look up client_auth0_application record, return if found.
**
*******************************************************************************/
QRecord getClientAuth0Application(Auth0AuthenticationMetaData metaData, String clientId) throws QException
static String maskForLog(String input)
{
//////////////////////////////////////////////////////////////////////////////////////
// try to look up existing auth0 application from database, insert one if not found //
//////////////////////////////////////////////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(metaData.getClientAuth0ApplicationTableName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(metaData.getAuth0ClientIdField(), QCriteriaOperator.EQUALS, clientId)));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
if(input == null)
{
return (queryOutput.getRecords().get(0));
return (null);
}
throw (new AccessTokenException("This client has not been configured to use the API.", HttpStatus.SC_UNAUTHORIZED));
if(input.length() < 8)
{
return ("******");
}
else
{
return (input.substring(0, 6) + "******");
}
}
}

73
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/metadata/UserSessionMetaDataProducer.java Normal file
View File

@ -0,0 +1,73 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.modules.authentication.implementations.metadata;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.model.UserSession;
/*******************************************************************************
** Meta Data Producer for UserSession
*******************************************************************************/
public class UserSessionMetaDataProducer extends MetaDataProducer<QTableMetaData>
{
private final String backendName;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public UserSessionMetaDataProducer(String backendName)
{
this.backendName = backendName;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QTableMetaData produce(QInstance qInstance) throws QException
{
QTableMetaData tableMetaData = new QTableMetaData()
.withName(UserSession.TABLE_NAME)
.withBackendName(backendName)
.withRecordLabelFormat("%s")
.withRecordLabelFields("id")
.withPrimaryKeyField("id")
.withUniqueKey(new UniqueKey("uuid"))
.withFieldsFromEntity(UserSession.class)
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE));
return tableMetaData;
}
}

262
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/model/UserSession.java Normal file
View File

@ -0,0 +1,262 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.modules.authentication.implementations.model;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
/*******************************************************************************
** QRecord Entity for UserSession table
*******************************************************************************/
public class UserSession extends QRecordEntity
{
public static final String TABLE_NAME = "userSession";
@QField(isEditable = false)
private Integer id;
@QField(isEditable = false)
private Instant createDate;
@QField(isEditable = false)
private Instant modifyDate;
@QField(isEditable = false, isHidden = true, maxLength = 40, valueTooLongBehavior = ValueTooLongBehavior.ERROR)
private String uuid;
@QField(isEditable = false, isHidden = true)
private String accessToken;
@QField(isEditable = false, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS)
private String userId;
/*******************************************************************************
** Default constructor
*******************************************************************************/
public UserSession()
{
}
/*******************************************************************************
** Constructor that takes a QRecord
*******************************************************************************/
public UserSession(QRecord record)
{
populateFromQRecord(record);
}
/*******************************************************************************
** Getter for id
*******************************************************************************/
public Integer getId()
{
return (this.id);
}
/*******************************************************************************
** Setter for id
*******************************************************************************/
public void setId(Integer id)
{
this.id = id;
}
/*******************************************************************************
** Fluent setter for id
*******************************************************************************/
public UserSession withId(Integer id)
{
this.id = id;
return (this);
}
/*******************************************************************************
** Getter for createDate
*******************************************************************************/
public Instant getCreateDate()
{
return (this.createDate);
}
/*******************************************************************************
** Setter for createDate
*******************************************************************************/
public void setCreateDate(Instant createDate)
{
this.createDate = createDate;
}
/*******************************************************************************
** Fluent setter for createDate
*******************************************************************************/
public UserSession withCreateDate(Instant createDate)
{
this.createDate = createDate;
return (this);
}
/*******************************************************************************
** Getter for modifyDate
*******************************************************************************/
public Instant getModifyDate()
{
return (this.modifyDate);
}
/*******************************************************************************
** Setter for modifyDate
*******************************************************************************/
public void setModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
}
/*******************************************************************************
** Fluent setter for modifyDate
*******************************************************************************/
public UserSession withModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
return (this);
}
/*******************************************************************************
** Getter for uuid
*******************************************************************************/
public String getUuid()
{
return (this.uuid);
}
/*******************************************************************************
** Setter for uuid
*******************************************************************************/
public void setUuid(String uuid)
{
this.uuid = uuid;
}
/*******************************************************************************
** Fluent setter for uuid
*******************************************************************************/
public UserSession withUuid(String uuid)
{
this.uuid = uuid;
return (this);
}
/*******************************************************************************
** Getter for accessToken
*******************************************************************************/
public String getAccessToken()
{
return (this.accessToken);
}
/*******************************************************************************
** Setter for accessToken
*******************************************************************************/
public void setAccessToken(String accessToken)
{
this.accessToken = accessToken;
}
/*******************************************************************************
** Fluent setter for accessToken
*******************************************************************************/
public UserSession withAccessToken(String accessToken)
{
this.accessToken = accessToken;
return (this);
}
/*******************************************************************************
** Getter for userId
*******************************************************************************/
public String getUserId()
{
return (this.userId);
}
/*******************************************************************************
** Setter for userId
*******************************************************************************/
public void setUserId(String userId)
{
this.userId = userId;
}
/*******************************************************************************
** Fluent setter for userId
*******************************************************************************/
public UserSession withUserId(String userId)
{
this.userId = userId;
return (this);
}
}

53
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java
View File

@ -25,11 +25,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryFilterLink;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@ -42,7 +46,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.scripts.Script;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptLog;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
@ -121,16 +127,25 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
getInput.setTableName(Script.TABLE_NAME);
getInput.setPrimaryKey(scriptId);
GetOutput getOutput = new GetAction().execute(getInput);
if(getOutput.getRecord() == null)
QRecord script = getOutput.getRecord();
if(script == null)
{
throw (new QException("Could not find script by id: " + scriptId));
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set an "audit context" - so any DML executed during the script will include the note of what script was running. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"" + script.getValue("name") + "\"");
String tableName = script.getValueString("tableName");
RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
input.setRecordList(runBackendStepInput.getRecords());
input.setCodeReference(new AdHocScriptCodeReference().withScriptId(scriptId));
input.setTableName(getOutput.getRecord().getValueString("tableName"));
input.setTableName(tableName);
input.setLogger(scriptLogger);
RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
Exception caughtException = null;
try
@ -147,11 +162,17 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
caughtException = e;
}
String auditMessage = "Script \"" + script.getValueString("name") + "\" (id: " + scriptId + ") was executed against this record";
//////////////////////////////////////////////////////////
// add the record to the appropriate processSummaryLine //
//////////////////////////////////////////////////////////
if(scriptLogger.getScriptLog() != null)
{
Integer id = scriptLogger.getScriptLog().getValueInteger("id");
if(id != null)
{
auditMessage += ", creating script log: " + id;
boolean hadError = BooleanUtils.isTrue(scriptLogger.getScriptLog().getValueBoolean("hadError"));
(hadError ? errorScriptLogIds : okScriptLogIds).add(id);
}
@ -160,6 +181,34 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
{
unloggedExceptionLine.incrementCount(runBackendStepInput.getRecords().size());
}
////////////////////////////////////////////////////////////
// audit that the script was executed against the records //
////////////////////////////////////////////////////////////
audit(runBackendStepInput, tableName, auditMessage);
}
/*******************************************************************************
** for each input record, add an audit stating that the script was executed.
*******************************************************************************/
private static void audit(RunBackendStepInput runBackendStepInput, String tableName, String auditMessage)
{
try
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
AuditInput auditInput = new AuditInput();
for(QRecord record : runBackendStepInput.getRecords())
{
AuditAction.appendToInput(auditInput, table, record, auditMessage);
}
new AuditAction().execute(auditInput);
}
catch(Exception e)
{
LOG.warn("Error recording audits after running record script", e, logPair("tableName", tableName), logPair("auditMessage", auditMessage));
}
}

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java
View File

@ -663,4 +663,19 @@ public class CollectionUtils
return (output);
}
/*******************************************************************************
**
*******************************************************************************/
public static <K> boolean containsKeyWithNonNullValue(Map<K, ?> map, K key)
{
if(map == null)
{
return (false);
}
return (map.containsKey(key) && map.get(key) != null);
}
}