Compare commits

...

10 Commits

11 changed files with 248 additions and 39 deletions

View File

@ -91,20 +91,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
long start = System.currentTimeMillis();
DMLType dmlType = getDMLType(tableActionInput);
///////////////////////////////////////////////////////////////////////////////////////////////////////
// currently, the table's primary key must be integer... so, log (once) and return early if not that //
///////////////////////////////////////////////////////////////////////////////////////////////////////
QFieldMetaData field = table.getField(table.getPrimaryKeyField());
if(!QFieldType.INTEGER.equals(field.getType()))
{
if(!loggedUnauditableTableNames.contains(table.getName()))
{
LOG.info("Cannot audit table without integer as its primary key", logPair("tableName", table.getName()));
loggedUnauditableTableNames.add(table.getName());
}
return (output);
}
try
{
List<QRecord> recordList = CollectionUtils.nonNullList(input.getRecordList()).stream()
@ -119,6 +105,21 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
return (output);
}
///////////////////////////////////////////////////////////////////////////////////////////////////////
// currently, the table's primary key must be integer... so, log (once) and return early if not that //
// (or, if no primary key!) //
///////////////////////////////////////////////////////////////////////////////////////////////////////
QFieldMetaData field = table.getFields().get(table.getPrimaryKeyField());
if(field == null || !QFieldType.INTEGER.equals(field.getType()))
{
if(!loggedUnauditableTableNames.contains(table.getName()))
{
LOG.info("Cannot audit table without integer as its primary key", logPair("tableName", table.getName()));
loggedUnauditableTableNames.add(table.getName());
}
return (output);
}
String contextSuffix = getContentSuffix(input);
AuditInput auditInput = new AuditInput();

View File

@ -31,6 +31,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Stream;
@ -1244,7 +1245,25 @@ public class QInstanceValidator
{
if(fieldMetaData.getDefaultValue() != null && fieldMetaData.getDefaultValue() instanceof QCodeReference codeReference)
{
validateSimpleCodeReference("Process " + processName + " backend step code reference: ", codeReference, BackendStep.class);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// by default, assume that any process field which is a QCodeReference should be a reference to a BackendStep... //
// but... allow a secondary field name to be set, to tell us what class to *actually* expect here... //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Class<?> expectedClass = BackendStep.class;
try
{
Optional<QFieldMetaData> expectedTypeField = backendStepMetaData.getInputMetaData().getField(fieldMetaData.getName() + "_expectedType");
if(expectedTypeField.isPresent() && expectedTypeField.get().getDefaultValue() != null)
{
expectedClass = Class.forName(ValueUtils.getValueAsString(expectedTypeField.get().getDefaultValue()));
}
}
catch(Exception e)
{
warn("Error loading expectedType for field [" + fieldMetaData.getName() + "] in process [" + processName + "]: " + e.getMessage());
}
validateSimpleCodeReference("Process " + processName + " code reference: ", codeReference, expectedClass);
}
}
}

View File

@ -746,12 +746,22 @@ public class QInstance
/*******************************************************************************
** Setter for hasBeenValidated
** If pass a QInstanceValidationKey (which can only be instantiated by the validator),
** then the hasBeenValidated field will be set to true.
**
** Else, if passed a null, hasBeenValidated will be reset to false - e.g., to
** re-trigger validation (can be useful in tests).
*******************************************************************************/
public void setHasBeenValidated(QInstanceValidationKey key)
{
this.hasBeenValidated = true;
if(key == null)
{
this.hasBeenValidated = false;
}
else
{
this.hasBeenValidated = true;
}
}

View File

@ -276,6 +276,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
if(sourceKeyValue == null || "".equals(sourceKeyValue))
{
LOG.debug("Skipping record without a value in the sourceKeyField", logPair("keyField", sourceTableKeyField));
errorMissingKeyField.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
try

View File

@ -176,6 +176,19 @@ public class StringUtils
/*******************************************************************************
** safely appends a string to another, changing empty string if either value is null
**
*******************************************************************************/
public static String safeAppend(String input, String contentToAppend)
{
input = input != null ? input : "";
contentToAppend = contentToAppend != null ? contentToAppend : "";
return input + contentToAppend;
}
/*******************************************************************************
** returns input if not null, or nullOutput if input == null (as in SQL NVL)
**

View File

@ -400,4 +400,49 @@ class DMLAuditActionTest extends BaseTest
QContext.popAction();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testTableWithoutIntegerPrimaryKey() throws QException
{
QInstance qInstance = QContext.getQInstance();
new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
////////////////////////////////////////////////////////////////////////////////////////////////////
// we used to throw if table had no primary key. first, assert that we do not throw in that case //
////////////////////////////////////////////////////////////////////////////////////////////////////
QContext.getQInstance().addTable(
new QTableMetaData()
.withName("nullPkey")
.withField(new QFieldMetaData("foo", QFieldType.STRING))
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)));
new DMLAuditAction().execute(new DMLAuditInput()
.withTableActionInput(new InsertInput("nullPkey"))
.withRecordList(List.of(new QRecord())));
//////////////////////////////////////////////////////////////////////////////////////////////
// next, make sure we don't throw (and don't record anything) if table's pkey isn't integer //
//////////////////////////////////////////////////////////////////////////////////////////////
QContext.getQInstance().addTable(
new QTableMetaData()
.withName("stringPkey")
.withField(new QFieldMetaData("idString", QFieldType.STRING))
.withPrimaryKeyField("idString")
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)));
new DMLAuditAction().execute(new DMLAuditInput()
.withTableActionInput(new InsertInput("stringPkey"))
.withRecordList(List.of(new QRecord())));
//////////////////////////////////
// make sure no audits happened //
//////////////////////////////////
List<QRecord> auditList = TestUtils.queryTable("audit");
assertTrue(auditList.isEmpty());
}
}

View File

@ -709,11 +709,11 @@ public class BaseAPIActionUtil
if(backendMetaData.getAuthorizationType().equals(AuthorizationType.BASIC_AUTH_USERNAME_PASSWORD))
{
request.addHeader("Authorization", getBasicAuthenticationHeader(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
request.setHeader("Authorization", getBasicAuthenticationHeader(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
}
else if(backendMetaData.getAuthorizationType().equals(AuthorizationType.API_KEY_HEADER))
{
request.addHeader("API-Key", record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
request.setHeader("API-Key", record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
}
else
{
@ -727,10 +727,10 @@ public class BaseAPIActionUtil
///////////////////////////////////////////////////////////////////////////////////////////
switch(backendMetaData.getAuthorizationType())
{
case BASIC_AUTH_API_KEY -> request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getApiKey()));
case BASIC_AUTH_USERNAME_PASSWORD -> request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getUsername(), backendMetaData.getPassword()));
case API_KEY_HEADER -> request.addHeader("API-Key", backendMetaData.getApiKey());
case API_TOKEN -> request.addHeader("Authorization", "Token " + backendMetaData.getApiKey());
case BASIC_AUTH_API_KEY -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getApiKey()));
case BASIC_AUTH_USERNAME_PASSWORD -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getUsername(), backendMetaData.getPassword()));
case API_KEY_HEADER -> request.setHeader("API-Key", backendMetaData.getApiKey());
case API_TOKEN -> request.setHeader("Authorization", "Token " + backendMetaData.getApiKey());
case OAUTH2 -> request.setHeader("Authorization", "Bearer " + getOAuth2Token());
case API_KEY_QUERY_PARAM ->
{
@ -786,9 +786,9 @@ public class BaseAPIActionUtil
if(setCredentialsInHeader)
{
request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
}
request.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8");
request.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8");
HttpResponse response = executeOAuthTokenRequest(client, request);
int statusCode = response.getStatusLine().getStatusCode();
@ -850,7 +850,7 @@ public class BaseAPIActionUtil
*******************************************************************************/
protected void setupContentTypeInRequest(HttpRequestBase request)
{
request.addHeader("Content-Type", backendMetaData.getContentType());
request.setHeader("Content-Type", backendMetaData.getContentType());
}
@ -872,7 +872,7 @@ public class BaseAPIActionUtil
*******************************************************************************/
public void setupAdditionalHeaders(HttpRequestBase request)
{
request.addHeader("Accept", "application/json");
request.setHeader("Accept", "application/json");
}

View File

@ -141,28 +141,38 @@ public class FilesystemImporterMetaDataTemplate
/*******************************************************************************
**
** Set up importRecord table being built by this template to hve an automation-
** status field on it, and an automation details object attached to it.
*******************************************************************************/
public void addAutomationStatusField(QTableMetaData table, QFieldMetaData automationStatusField)
public void addImportRecordAutomations(QFieldMetaData automationStatusField, QTableAutomationDetails automationDetails)
{
table.addField(automationStatusField);
table.getSections().get(1).getFieldNames().add(0, automationStatusField.getName());
getImportRecordTable().addField(automationStatusField);
getImportRecordTable().getSections().get(1).getFieldNames().add(0, automationStatusField.getName());
getImportRecordTable().withAutomationDetails(automationDetails);
}
/*******************************************************************************
** Add 1 process as a post-insert automation-action on this template's importRecord
** table.
**
** The automation action is returned - which you may want for changing things, e.g.,
** its priority (e.g., addImportRecordPostInsertAutomationAction(...).withPriority(1);
*******************************************************************************/
public TableAutomationAction addStandardPostInsertAutomation(QTableMetaData table, QTableAutomationDetails automationDetails, String processName)
public TableAutomationAction addImportRecordPostInsertAutomationAction(String processName)
{
if(getImportRecordTable().getAutomationDetails() == null)
{
throw (new IllegalStateException(getImportRecordTable().getName() + " does not have automationDetails - do you need to call addAutomations first?"));
}
TableAutomationAction action = new TableAutomationAction()
.withName(table.getName() + "PostInsert")
.withName(processName)
.withTriggerEvent(TriggerEvent.POST_INSERT)
.withProcessName(processName);
table.withAutomationDetails(automationDetails
.withAction(action));
getImportRecordTable().getAutomationDetails().withAction(action);
return (action);
}

View File

@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.io.Serializable;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@ -62,6 +64,15 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_PATH, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, QFieldType.STRING))
//////////////////////////////////////////////////////////////////////////////////////
// define a QCodeReference - expected to be of type Function<QRecord, Serializable> //
// make sure the QInstanceValidator knows that the QCodeReference should be a //
// Function (not a BackendStep, which is the default for process fields) //
//////////////////////////////////////////////////////////////////////////////////////
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_VALUE_SUPPLIER, QFieldType.STRING))
.withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_VALUE_SUPPLIER + "_expectedType", QFieldType.STRING)
.withDefaultValue(Function.class.getName()))
)));
}
@ -186,4 +197,15 @@ public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMet
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public FilesystemImporterProcessMetaDataBuilder withImportSecurityValueSupplierFunction(Class<? extends Function<QRecord, Serializable>> supplierFunction)
{
setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_VALUE_SUPPLIER, new QCodeReference(supplierFunction));
return (this);
}
}

View File

@ -33,7 +33,9 @@ import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.UUID;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@ -41,6 +43,7 @@ import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@ -53,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -83,8 +87,9 @@ public class FilesystemImporterStep implements BackendStep
public static final String FIELD_IMPORT_FILE_TABLE = "importFileTable";
public static final String FIELD_IMPORT_RECORD_TABLE = "importRecordTable";
public static final String FIELD_IMPORT_SECURITY_FIELD_NAME = "importSecurityFieldName";
public static final String FIELD_IMPORT_SECURITY_FIELD_VALUE = "importSecurityFieldValue";
public static final String FIELD_IMPORT_SECURITY_FIELD_NAME = "importSecurityFieldName";
public static final String FIELD_IMPORT_SECURITY_FIELD_VALUE = "importSecurityFieldValue";
public static final String FIELD_IMPORT_SECURITY_VALUE_SUPPLIER = "importSecurityFieldSupplier";
public static final String FIELD_ARCHIVE_FILE_ENABLED = "archiveFileEnabled";
public static final String FIELD_ARCHIVE_TABLE_NAME = "archiveTableName";
@ -93,6 +98,7 @@ public class FilesystemImporterStep implements BackendStep
public static final String FIELD_UPDATE_FILE_IF_NAME_EXISTS = "updateFileIfNameExists";
private Function<QRecord, Serializable> securitySupplier = null;
/*******************************************************************************
@ -267,9 +273,34 @@ public class FilesystemImporterStep implements BackendStep
*******************************************************************************/
private void addSecurityValue(RunBackendStepInput runBackendStepInput, QRecord record)
{
String securityField = runBackendStepInput.getValueString(FIELD_IMPORT_SECURITY_FIELD_NAME);
Serializable securityValue = runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_FIELD_VALUE);
String securityField = runBackendStepInput.getValueString(FIELD_IMPORT_SECURITY_FIELD_NAME);
/////////////////////////////////////////////////////////////
// if we're using a security supplier function, load it up //
/////////////////////////////////////////////////////////////
QCodeReference securitySupplierReference = (QCodeReference) runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_VALUE_SUPPLIER);
try
{
if(securitySupplierReference != null && securitySupplier == null)
{
securitySupplier = QCodeLoader.getAdHoc(Function.class, securitySupplierReference);
}
}
catch(Exception e)
{
throw (new QRuntimeException("Error loading Security Supplier Function from QCodeReference [" + securitySupplierReference + "]", e));
}
///////////////////////////////////////////////////////////////////////////////////////
// either get the security value from the supplier, or the field value field's value //
///////////////////////////////////////////////////////////////////////////////////////
Serializable securityValue = securitySupplier != null
? securitySupplier.apply(record)
: runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_FIELD_VALUE);
////////////////////////////////////////////////////////////////////
// if we have a field name and a value, then add it to the record //
////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(securityField) && securityValue != null)
{
record.setValue(securityField, securityValue);

View File

@ -23,16 +23,20 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.io.File;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -261,4 +265,57 @@ class FilesystemImporterStepTest extends FilesystemActionTest
assertEquals(47, recordRecord.getValue("customerId"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSecuritySupplier() throws QException
{
//////////////////////////////////////////////
// Add a security name/value to our process //
//////////////////////////////////////////////
QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME)).findFirst().get().setDefaultValue("customerId");
process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_VALUE_SUPPLIER)).findFirst().get().setDefaultValue(new QCodeReference(SecuritySupplier.class));
//////////////////////////////////////////////////////////////////////////////////////////////////////
// re-validate our instance now that we have that code-reference in place for the security supplier //
//////////////////////////////////////////////////////////////////////////////////////////////////////
QContext.getQInstance().setHasBeenValidated(null);
new QInstanceValidator().validate(QContext.getQInstance());
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
new RunProcessAction().execute(runProcessInput);
////////////////////////////////////////////////////////////////////////////////////////////
// assert the security field gets its value on both the importFile & importRecord records //
////////////////////////////////////////////////////////////////////////////////////////////
String importBaseName = "personImporter";
QRecord fileRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX).withPrimaryKey(1));
assertEquals(1701, fileRecord.getValue("customerId"));
QRecord recordRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
assertEquals(1701, recordRecord.getValue("customerId"));
}
/*******************************************************************************
**
*******************************************************************************/
public static class SecuritySupplier implements Function<QRecord, Serializable>
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public Serializable apply(QRecord qRecord)
{
return (1701);
}
}
}