Compare commits

...

10 Commits

11 changed files with 366 additions and 27 deletions

View File

@ -225,7 +225,13 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
{
if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
{
throw (new QException("Missing securityKeyValue [" + recordSecurityLock.getSecurityKeyType() + "] in audit request for table " + auditSingleInput.getAuditTableName()));
///////////////////////////////////////////////////////
// originally, this case threw... //
// but i think it's better to record the audit, just //
// missing its security key value, then to fail... //
///////////////////////////////////////////////////////
// throw (new QException("Missing securityKeyValue [" + recordSecurityLock.getSecurityKeyType() + "] in audit request for table " + auditSingleInput.getAuditTableName()));
LOG.info("Missing securityKeyValue in audit request", logPair("table", auditSingleInput.getAuditTableName()), logPair("securityKey", recordSecurityLock.getSecurityKeyType()));
}
}

View File

@ -79,6 +79,7 @@ public class RunProcessAction
{
private static final QLogger LOG = QLogger.getLogger(RunProcessAction.class);
public static final String BASEPULL_KEY_VALUE = "basepullKeyValue";
public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
public static final String BASEPULL_TIMESTAMP_FIELD = "basepullTimestampField";
@ -517,9 +518,13 @@ public class RunProcessAction
/*******************************************************************************
**
*******************************************************************************/
protected String determineBasepullKeyValue(QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
protected String determineBasepullKeyValue(QProcessMetaData process, RunProcessInput runProcessInput, BasepullConfiguration basepullConfiguration) throws QException
{
String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
if(runProcessInput.getValueString(BASEPULL_KEY_VALUE) != null)
{
basepullKeyValue = runProcessInput.getValueString(BASEPULL_KEY_VALUE);
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if process specifies that it uses variants, look for that data in the session and append to our basepull key //
@ -551,7 +556,7 @@ public class RunProcessAction
String basepullTableName = basepullConfiguration.getTableName();
String basepullKeyFieldName = basepullConfiguration.getKeyField();
String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName();
String basepullKeyValue = determineBasepullKeyValue(process, basepullConfiguration);
String basepullKeyValue = determineBasepullKeyValue(process, runProcessInput, basepullConfiguration);
///////////////////////////////////////
// get the stored basepull timestamp //
@ -631,7 +636,7 @@ public class RunProcessAction
String basepullKeyFieldName = basepullConfiguration.getKeyField();
String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName();
Integer basepullHoursBackForInitialTimestamp = basepullConfiguration.getHoursBackForInitialTimestamp();
String basepullKeyValue = determineBasepullKeyValue(process, basepullConfiguration);
String basepullKeyValue = determineBasepullKeyValue(process, runProcessInput, basepullConfiguration);
///////////////////////////////////////
// get the stored basepull timestamp //

View File

@ -260,9 +260,6 @@ public class SearchPossibleValueSourceAction
}
}
// todo - skip & limit as params
queryFilter.setLimit(250);
///////////////////////////////////////////////////////////////////////////////////////////////////////
// if given a default filter, make it the 'top level' filter and the one we just created a subfilter //
///////////////////////////////////////////////////////////////////////////////////////////////////////
@ -272,6 +269,9 @@ public class SearchPossibleValueSourceAction
queryFilter = input.getDefaultQueryFilter();
}
// todo - skip & limit as params
queryFilter.setLimit(250);
queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
queryInput.setFilter(queryFilter);

View File

@ -247,10 +247,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
//////////////////////////////////////////////////////////////
// allow customizer to do custom things here, if so desired //
//////////////////////////////////////////////////////////////
if(getCustomizer() != null)
{
getCustomizer().finalCustomizeSession(qInstance, qSession);
}
finalCustomizeSession(qInstance, qSession);
return (qSession);
}
@ -311,10 +308,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
//////////////////////////////////////////////////////////////
// allow customizer to do custom things here, if so desired //
//////////////////////////////////////////////////////////////
if(getCustomizer() != null)
{
getCustomizer().finalCustomizeSession(qInstance, qSession);
}
finalCustomizeSession(qInstance, qSession);
return (qSession);
}
@ -360,6 +354,23 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
/***************************************************************************
**
***************************************************************************/
private void finalCustomizeSession(QInstance qInstance, QSession qSession)
{
if(getCustomizer() != null)
{
QContext.withTemporaryContext(QContext.capture(), () ->
{
QContext.setQSession(getChickenAndEggSession());
getCustomizer().finalCustomizeSession(qInstance, qSession);
});
}
}
/*******************************************************************************
** Insert a session as a new record into userSession table
*******************************************************************************/

View File

@ -72,33 +72,33 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
*******************************************************************************/
public abstract class AbstractTableSyncTransformStep extends AbstractTransformStep
{
private static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class);
protected static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class);
private ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine();
private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine();
protected ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine();
protected ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine();
private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
protected ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
.withMessageSuffix("because this process is not configured to insert records.")
.withSingularFutureMessage("will not be inserted ")
.withPluralFutureMessage("will not be inserted ")
.withSingularPastMessage("was not inserted ")
.withPluralPastMessage("were not inserted ");
private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
protected ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
.withMessageSuffix("because this process is not configured to update records.")
.withSingularFutureMessage("will not be updated ")
.withPluralFutureMessage("will not be updated ")
.withSingularPastMessage("was not updated ")
.withPluralPastMessage("were not updated ");
private ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR)
protected ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR)
.withMessageSuffix("missing a value for the key field.")
.withSingularFutureMessage("will not be synced, because it is ")
.withPluralFutureMessage("will not be synced, because they are ")
.withSingularPastMessage("was not synced, because it is ")
.withPluralPastMessage("were not synced, because they are ");
private ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR)
protected ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR)
.withMessageSuffix("of an unexpected error: ")
.withSingularFutureMessage("will not be synced, ")
.withPluralFutureMessage("will not be synced, ")
@ -109,7 +109,10 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
protected RunBackendStepOutput runBackendStepOutput = null;
protected RecordLookupHelper recordLookupHelper = null;
private QPossibleValueTranslator possibleValueTranslator;
protected QPossibleValueTranslator possibleValueTranslator;
protected static final String SYNC_TABLE_PERFORM_INSERTS_KEY = "syncTablePerformInsertsKey";
protected static final String SYNC_TABLE_PERFORM_UPDATES_KEY = "syncTablePerformUpdatesKey";
@ -193,6 +196,26 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
/*******************************************************************************
**
*******************************************************************************/
void setPerformInserts(boolean performInserts)
{
this.setPerformInserts(performInserts);
}
/*******************************************************************************
**
*******************************************************************************/
void setPerformUpdates(boolean performUpdates)
{
this.setPerformUpdates(performUpdates);
}
/*******************************************************************************
** artificial method, here to make jacoco see that this class is indeed
** included in test coverage...
@ -222,6 +245,18 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
SyncProcessConfig config = getSyncProcessConfig();
////////////////////////////////////////////////////////////
// see if these fields have been updated via input fields //
////////////////////////////////////////////////////////////
if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY) != null)
{
config.setPerformInserts(Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY)));
}
if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY) != null)
{
config.setPerformUpdates(Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY)));
}
String sourceTableKeyField = config.sourceTableKeyField;
String destinationTableForeignKeyField = config.destinationTableForeignKey;
String destinationTableName = config.destinationTable;
@ -374,6 +409,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
}
/*******************************************************************************
** Given a source record, extract what we'll use as its key from it.
**

View File

@ -30,6 +30,8 @@ import java.util.Optional;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
@ -40,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -83,6 +86,7 @@ class AuditActionTest extends BaseTest
**
*******************************************************************************/
@Test
@Disabled("this behavior has been changed to just log... should this be a setting?")
void testFailWithoutSecurityKey() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
@ -117,6 +121,50 @@ class AuditActionTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testLogWithoutSecurityKey() throws QException
{
int recordId = 1701;
QInstance qInstance = TestUtils.defineInstance();
new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
String userName = "John Doe";
QContext.init(qInstance, new QSession().withUser(new QUser().withFullName(userName)));
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(AuditAction.class);
AuditAction.execute(TestUtils.TABLE_NAME_ORDER, recordId, Map.of(), "Test Audit");
///////////////////////////////////////////////////////////////////
// it should not throw, but it should also not insert the audit. //
///////////////////////////////////////////////////////////////////
Optional<QRecord> auditRecord = GeneralProcessUtils.getRecordByField("audit", "recordId", recordId);
assertTrue(auditRecord.isPresent());
assertThat(collectingLogger.getCollectedMessages()).anyMatch(m -> m.getMessage().contains("Missing securityKeyValue in audit request"));
collectingLogger.clear();
////////////////////////////////////////////////////////////////////////////////////////////////
// try again with a null value in the key - that should be ok - as at least you were thinking //
// about the key and put in SOME value (null has its own semantics in security keys) //
////////////////////////////////////////////////////////////////////////////////////////////////
Map<String, Serializable> securityKeys = new HashMap<>();
securityKeys.put(TestUtils.SECURITY_KEY_TYPE_STORE, null);
AuditAction.execute(TestUtils.TABLE_NAME_ORDER, recordId, securityKeys, "Test Audit");
/////////////////////////////////////
// now the audit should be stored. //
/////////////////////////////////////
auditRecord = GeneralProcessUtils.getRecordByField("audit", "recordId", recordId);
assertTrue(auditRecord.isPresent());
assertThat(collectingLogger.getCollectedMessages()).noneMatch(m -> m.getMessage().contains("Missing securityKeyValue in audit request"));
}
/*******************************************************************************
**
*******************************************************************************/

View File

@ -748,7 +748,7 @@ public class BaseAPIActionUtil
{
try
{
String uri = request.getURI().toString();
String uri = request.getURI().toString();
String pair = backendMetaData.getApiKeyQueryParamName() + "=" + getApiKey();
///////////////////////////////////////////////////////////////////////////////////
@ -1167,6 +1167,16 @@ public class BaseAPIActionUtil
/*******************************************************************************
**
*******************************************************************************/
protected void logRequestDetails(QTableMetaData table, HttpRequestBase request) throws QException
{
LOG.info("Making [" + request.getMethod() + "] request to URL [" + request.getURI() + "] on table [" + table.getName() + "].");
}
/*******************************************************************************
**
*******************************************************************************/
@ -1192,7 +1202,7 @@ public class BaseAPIActionUtil
setupContentTypeInRequest(request);
setupAdditionalHeaders(request);
LOG.info("Making [" + request.getMethod() + "] request to URL [" + request.getURI() + "] on table [" + table.getName() + "].");
logRequestDetails(table, request);
if("POST".equals(request.getMethod()))
{
LOG.info("POST contents [" + ((HttpPost) request).getEntity().toString() + "]");

View File

@ -1279,6 +1279,11 @@ public class QJavalinImplementation
queryInput.getFilter().setLimit(limit);
}
if(queryInput.getFilter() == null || queryInput.getFilter().getLimit() == null)
{
handleQueryNullLimit(context, queryInput);
}
List<QueryJoin> queryJoins = processQueryJoinsParam(context);
queryInput.setQueryJoins(queryJoins);
@ -1299,6 +1304,28 @@ public class QJavalinImplementation
/***************************************************************************
**
***************************************************************************/
private static void handleQueryNullLimit(Context context, QueryInput queryInput)
{
boolean allowed = javalinMetaData.getQueryWithoutLimitAllowed();
if(!allowed)
{
if(queryInput.getFilter() == null)
{
queryInput.setFilter(new QQueryFilter());
}
queryInput.getFilter().setLimit(javalinMetaData.getQueryWithoutLimitDefault());
LOG.log(javalinMetaData.getQueryWithoutLimitLogLevel(), "Query request did not specify a limit, which is not allowed. Using default instead", null,
logPair("defaultLimit", javalinMetaData.getQueryWithoutLimitDefault()),
logPair("path", context.path()));
}
}
/*******************************************************************************
**
*******************************************************************************/

View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.javalin;
import java.util.function.Function;
import org.apache.logging.log4j.Level;
/*******************************************************************************
@ -36,6 +37,10 @@ public class QJavalinMetaData
private Function<QJavalinAccessLogger.LogEntry, Boolean> logFilter;
private boolean queryWithoutLimitAllowed = false;
private Integer queryWithoutLimitDefault = 1000;
private Level queryWithoutLimitLogLevel = Level.INFO;
/*******************************************************************************
@ -143,4 +148,97 @@ public class QJavalinMetaData
return (this);
}
/*******************************************************************************
** Getter for queryWithoutLimitAllowed
*******************************************************************************/
public boolean getQueryWithoutLimitAllowed()
{
return (this.queryWithoutLimitAllowed);
}
/*******************************************************************************
** Setter for queryWithoutLimitAllowed
*******************************************************************************/
public void setQueryWithoutLimitAllowed(boolean queryWithoutLimitAllowed)
{
this.queryWithoutLimitAllowed = queryWithoutLimitAllowed;
}
/*******************************************************************************
** Fluent setter for queryWithoutLimitAllowed
*******************************************************************************/
public QJavalinMetaData withQueryWithoutLimitAllowed(boolean queryWithoutLimitAllowed)
{
this.queryWithoutLimitAllowed = queryWithoutLimitAllowed;
return (this);
}
/*******************************************************************************
** Getter for queryWithoutLimitDefault
*******************************************************************************/
public Integer getQueryWithoutLimitDefault()
{
return (this.queryWithoutLimitDefault);
}
/*******************************************************************************
** Setter for queryWithoutLimitDefault
*******************************************************************************/
public void setQueryWithoutLimitDefault(Integer queryWithoutLimitDefault)
{
this.queryWithoutLimitDefault = queryWithoutLimitDefault;
}
/*******************************************************************************
** Fluent setter for queryWithoutLimitDefault
*******************************************************************************/
public QJavalinMetaData withQueryWithoutLimitDefault(Integer queryWithoutLimitDefault)
{
this.queryWithoutLimitDefault = queryWithoutLimitDefault;
return (this);
}
/*******************************************************************************
** Getter for queryWithoutLimitLogLevel
*******************************************************************************/
public Level getQueryWithoutLimitLogLevel()
{
return (this.queryWithoutLimitLogLevel);
}
/*******************************************************************************
** Setter for queryWithoutLimitLogLevel
*******************************************************************************/
public void setQueryWithoutLimitLogLevel(Level queryWithoutLimitLogLevel)
{
this.queryWithoutLimitLogLevel = queryWithoutLimitLogLevel;
}
/*******************************************************************************
** Fluent setter for queryWithoutLimitLogLevel
*******************************************************************************/
public QJavalinMetaData withQueryWithoutLimitLogLevel(Level queryWithoutLimitLogLevel)
{
this.queryWithoutLimitLogLevel = queryWithoutLimitLogLevel;
return (this);
}
}

View File

@ -33,6 +33,8 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@ -45,6 +47,7 @@ import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import kong.unirest.HttpResponse;
import kong.unirest.Unirest;
import org.apache.logging.log4j.Level;
import org.eclipse.jetty.http.HttpStatus;
import org.json.JSONArray;
import org.json.JSONObject;
@ -469,6 +472,101 @@ class QJavalinImplementationTest extends QJavalinTestBase
/*******************************************************************************
** test a table query using an actual filter via POST, with no limit specified,
** and with that not being allowed.
**
*******************************************************************************/
@Test
public void test_dataQueryWithFilterPOSTWithoutLimitNotAllowed() throws QInstanceValidationException
{
try
{
qJavalinImplementation.getJavalinMetaData()
.withQueryWithoutLimitAllowed(false)
.withQueryWithoutLimitDefault(3)
.withQueryWithoutLimitLogLevel(Level.WARN);
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QJavalinImplementation.class);
String filterJson = """
{"criteria":[]}""";
HttpResponse<String> response = Unirest.post(BASE_URL + "/data/person/query")
.field("filter", filterJson)
.asString();
assertEquals(200, response.getStatus());
JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
assertTrue(jsonObject.has("records"));
JSONArray records = jsonObject.getJSONArray("records");
assertEquals(3, records.length());
assertThat(collectingLogger.getCollectedMessages())
.anyMatch(m -> m.getLevel().equals(Level.WARN) && m.getMessage().contains("Query request did not specify a limit"));
}
finally
{
QLogger.activateCollectingLoggerForClass(QJavalinImplementation.class);
resetMetaDataQueryWithoutLimitSettings();
}
}
/*******************************************************************************
** test a table query using an actual filter via POST, with no limit specified,
** but with that being allowed.
**
*******************************************************************************/
@Test
public void test_dataQueryWithFilterPOSTWithoutLimitAllowed() throws QInstanceValidationException
{
try
{
qJavalinImplementation.getJavalinMetaData()
.withQueryWithoutLimitAllowed(true);
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QJavalinImplementation.class);
String filterJson = """
{"criteria":[]}""";
HttpResponse<String> response = Unirest.post(BASE_URL + "/data/person/query")
.field("filter", filterJson)
.asString();
assertEquals(200, response.getStatus());
JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
assertTrue(jsonObject.has("records"));
JSONArray records = jsonObject.getJSONArray("records");
assertEquals(6, records.length());
assertThat(collectingLogger.getCollectedMessages())
.noneMatch(m -> m.getMessage().contains("Query request did not specify a limit"));
}
finally
{
QLogger.activateCollectingLoggerForClass(QJavalinImplementation.class);
resetMetaDataQueryWithoutLimitSettings();
}
}
/***************************************************************************
**
***************************************************************************/
private void resetMetaDataQueryWithoutLimitSettings()
{
qJavalinImplementation.getJavalinMetaData()
.withQueryWithoutLimitAllowed(false)
.withQueryWithoutLimitDefault(1000)
.withQueryWithoutLimitLogLevel(Level.INFO);
}
/*******************************************************************************
**
*******************************************************************************/
@ -1029,7 +1127,7 @@ class QJavalinImplementationTest extends QJavalinTestBase
// filter use-case, with no values, should return options. //
/////////////////////////////////////////////////////////////
HttpResponse<String> response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=filter").asString();
JSONArray options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
JSONArray options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
assertNotNull(options);
assertThat(options.length()).isGreaterThanOrEqualTo(5);

View File

@ -102,7 +102,7 @@ public class QJavalinTestBase
{
qJavalinImplementation.stopJavalinServer();
}
qJavalinImplementation = new QJavalinImplementation(qInstance);
qJavalinImplementation = new QJavalinImplementation(qInstance, new QJavalinMetaData());
QJavalinProcessHandler.setAsyncStepTimeoutMillis(250);
qJavalinImplementation.startJavalinServer(PORT);
}