QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

more support for blobs

Browse Source 
- fetch heavy flag for process extract
- ignore blobs in bulk edit, load
- set download urls in child-list render and javalin via shared method in QValueFormatter

also, make ETL load step aware of transform step, and more flexible process summary between them
This commit is contained in:
darin.kelkhoff
2023-06-05 08:17:45 -05:00
parent 499d59ade2
commit fdbc751048
10 changed files with 227 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java
View File

@ -33,6 +33,7 @@ import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
@ -160,10 +161,12 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
String widgetLabel = input.getQueryParams().get("widgetLabel");
String joinName = input.getQueryParams().get("joinName");
QJoinMetaData join = input.getInstance().getJoin(joinName);
String id = input.getQueryParams().get("id");
String widgetLabel = input.getQueryParams().get("widgetLabel");
String joinName = input.getQueryParams().get("joinName");
QJoinMetaData join = input.getInstance().getJoin(joinName);
String id = input.getQueryParams().get("id");
QTableMetaData leftTable = input.getInstance().getTable(join.getLeftTable());
QTableMetaData rightTable = input.getInstance().getTable(join.getRightTable());
Integer maxRows = null;
if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
@ -187,8 +190,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
if(record == null)
{
QTableMetaData table = input.getInstance().getTable(join.getLeftTable());
throw (new QNotFoundException("Could not find " + (table == null ? "" : table.getLabel()) + " with primary key " + id));
throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
}
////////////////////////////////////////////////////////////////////
@ -209,6 +211,8 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
queryInput.setFilter(filter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
int totalRows = queryOutput.getRecords().size();
if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
{
@ -222,11 +226,10 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
totalRows = new CountAction().execute(countInput).getCount();
}
QTableMetaData table = input.getInstance().getTable(join.getRightTable());
String tablePath = input.getInstance().getTablePath(table.getName());
String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
String tablePath = input.getInstance().getTablePath(rightTable.getName());
String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, table, tablePath, viewAllLink, totalRows);
ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
{

128
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
View File

@ -29,13 +29,18 @@ import java.time.LocalTime;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -434,4 +439,127 @@ public class QValueFormatter
}
}
/*******************************************************************************
** For any BLOB type fields in the list of records, change their value to
** the URL where they can be downloaded, and set their display value to a file name.
*******************************************************************************/
public static void setBlobValuesToDownloadUrls(QTableMetaData table, List<QRecord> records)
{
for(QFieldMetaData field : table.getFields().values())
{
if(field.getType().equals(QFieldType.BLOB))
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// file name comes from: //
// if there's a FILE_DOWNLOAD adornment, with a FILE_NAME_FIELD value, then the full filename comes from that field //
// - unless it was empty - then we do the "default thing": //
// else - the "default thing" is: //
// - tableLabel primaryKey fieldLabel //
// - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot) //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Optional<FieldAdornment> fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
Map<String, Serializable> adornmentValues = Collections.emptyMap();
if(fileDownloadAdornment.isPresent())
{
adornmentValues = fileDownloadAdornment.get().getValues();
}
String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
for(QRecord record : records)
{
if(!doesFieldHaveValue(field, record))
{
continue;
}
Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
String fileName = null;
//////////////////////////////////////////////////
// try to make file name from the fileNameField //
//////////////////////////////////////////////////
if(StringUtils.hasContent(fileNameField))
{
fileName = record.getValueString(fileNameField);
}
if(!StringUtils.hasContent(fileName))
{
if(StringUtils.hasContent(fileNameFormat))
{
@SuppressWarnings("unchecked") // instance validation should make this safe!
List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
}
}
if(!StringUtils.hasContent(fileName))
{
//////////////////////////////////
// make default name if missing //
//////////////////////////////////
fileName = table.getLabel() + " " + primaryKey + " " + field.getLabel();
if(StringUtils.hasContent(defaultExtension))
{
//////////////////////////////////////////
// add default extension if we have one //
//////////////////////////////////////////
fileName += "." + defaultExtension;
}
}
record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
record.setDisplayValue(field.getName(), fileName);
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record)
{
boolean fieldHasValue = false;
try
{
if(record.getValue(field.getName()) != null)
{
fieldHasValue = true;
}
else if(field.getIsHeavy())
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
if(heavyFieldLengths != null)
{
Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName()));
if(fieldLength != null && fieldLength > 0)
{
fieldHasValue = true;
}
}
}
}
catch(Exception e)
{
LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record));
}
return fieldHasValue;
}
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -716,7 +716,7 @@ public class QInstanceEnricher
try
{
QFieldMetaData field = table.getField(fieldName);
if(field.getIsEditable())
if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB))
{
editableFields.add(field);
}
@ -774,6 +774,7 @@ public class QInstanceEnricher
List<QFieldMetaData> editableFields = table.getFields().values().stream()
.filter(QFieldMetaData::getIsEditable)
.filter(f -> !f.getType().equals(QFieldType.BLOB))
.toList();
QFrontendStepMetaData editScreen = new QFrontendStepMetaData()

22
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java
View File

@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -290,4 +291,25 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public void addAuditSingleInput(AuditSingleInput auditSingleInput)
{
if(getAuditInputList() == null)
{
setAuditInputList(new ArrayList<>());
}
if(getAuditInputList().isEmpty())
{
getAuditInputList().add(new AuditInput());
}
AuditInput auditInput = getAuditInputList().get(0);
auditInput.addAuditSingleInput(auditSingleInput);
}
}

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java
View File

@ -70,6 +70,17 @@ public class AbstractProcessMetaDataBuilder
/*******************************************************************************
**
*******************************************************************************/
public AbstractProcessMetaDataBuilder withInputFieldDefaultValue(String fieldName, Serializable value)
{
setInputFieldDefaultValue(fieldName, value);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java
View File

@ -47,6 +47,8 @@ public abstract class AbstractLoadStep implements BackendStep
private Optional<QBackendTransaction> transaction = Optional.empty();
protected QSession session;
private AbstractTransformStep transformStep;
/*******************************************************************************
@ -122,4 +124,35 @@ public abstract class AbstractLoadStep implements BackendStep
return (null);
}
/*******************************************************************************
** Getter for transformStep
*******************************************************************************/
public AbstractTransformStep getTransformStep()
{
return (this.transformStep);
}
/*******************************************************************************
** Setter for transformStep
*******************************************************************************/
public void setTransformStep(AbstractTransformStep transformStep)
{
this.transformStep = transformStep;
}
/*******************************************************************************
** Fluent setter for transformStep
*******************************************************************************/
public AbstractLoadStep withTransformStep(AbstractTransformStep transformStep)
{
this.transformStep = transformStep;
return (this);
}
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java
View File

@ -109,6 +109,11 @@ public class ExtractViaQueryStep extends AbstractExtractStep
queryInput.setRecordPipe(getRecordPipe());
queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
if(runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_FETCH_HEAVY_FIELDS))
{
queryInput.setShouldFetchHeavyFields(true);
}
customizeInputPreQuery(queryInput);
new QueryAction().execute(queryInput);

12
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
View File

@ -73,6 +73,8 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
AbstractLoadStep loadStep = getLoadStep(runBackendStepInput);
loadStep.setTransformStep(transformStep);
/////////////////////////////////////////////////////////////////////////////
// let the load step override the capacity for the record pipe. //
// this is useful for slower load steps - so that the extract step doesn't //
@ -140,13 +142,17 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// get the process summary from the load step, if it's a summary-provider -- else, use the transform step (which is always a provider) //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ArrayList<ProcessSummaryLineInterface> processSummaryLines = null;
if(loadStep instanceof ProcessSummaryProviderInterface provider)
{
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, provider.doGetProcessSummary(runBackendStepOutput, true));
processSummaryLines = provider.doGetProcessSummary(runBackendStepOutput, true);
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryLines);
}
else
if(CollectionUtils.nullSafeIsEmpty(processSummaryLines))
{
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, transformStep.doGetProcessSummary(runBackendStepOutput, true));
processSummaryLines = transformStep.doGetProcessSummary(runBackendStepOutput, true);
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryLines);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java
View File

@ -81,6 +81,7 @@ public class StreamedETLWithFrontendProcess
public static final String FIELD_DESTINATION_TABLE = "destinationTable"; // String
public static final String FIELD_RECORD_COUNT = "recordCount"; // Integer
public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter)
public static final String FIELD_FETCH_HEAVY_FIELDS = "fetchHeavyFields"; // Boolean
public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean
public static final String FIELD_DO_FULL_VALIDATION = "doFullValidation"; // Boolean
@ -142,6 +143,7 @@ public class StreamedETLWithFrontendProcess
.withCode(new QCodeReference(StreamedETLPreviewStep.class))
.withInputData(new QFunctionInputMetaData()
.withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE)))
.withField(new QFieldMetaData(FIELD_FETCH_HEAVY_FIELDS, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_FETCH_HEAVY_FIELDS, false)))
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE)))
.withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, true)))
.withField(new QFieldMetaData(FIELD_DO_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.get(FIELD_DO_FULL_VALIDATION)))

128
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -32,7 +32,6 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@ -858,7 +857,7 @@ public class QJavalinImplementation
+ table.getFields().get(table.getPrimaryKeyField()).getLabel() + " of " + primaryKey));
}
setBlobValuesToDownloadUrls(table, List.of(record));
QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record));
QJavalinAccessLogger.logEndSuccess();
context.result(JsonUtils.toJson(record));
@ -872,129 +871,6 @@ public class QJavalinImplementation
/*******************************************************************************
** For any BLOB type fields in the list of records, change their value to
** the URL where they can be downloaded, and set their display value to a file name.
*******************************************************************************/
private static void setBlobValuesToDownloadUrls(QTableMetaData table, List<QRecord> records)
{
for(QFieldMetaData field : table.getFields().values())
{
if(field.getType().equals(QFieldType.BLOB))
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// file name comes from: //
// if there's a FILE_DOWNLOAD adornment, with a FILE_NAME_FIELD value, then the full filename comes from that field //
// - unless it was empty - then we do the "default thing": //
// else - the "default thing" is: //
// - tableLabel primaryKey fieldLabel //
// - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot) //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Optional<FieldAdornment> fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
Map<String, Serializable> adornmentValues = Collections.emptyMap();
if(fileDownloadAdornment.isPresent())
{
adornmentValues = fileDownloadAdornment.get().getValues();
}
String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
for(QRecord record : records)
{
if(!doesFieldHaveValue(field, record))
{
continue;
}
Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
String fileName = null;
//////////////////////////////////////////////////
// try to make file name from the fileNameField //
//////////////////////////////////////////////////
if(StringUtils.hasContent(fileNameField))
{
fileName = record.getValueString(fileNameField);
}
if(!StringUtils.hasContent(fileName))
{
if(StringUtils.hasContent(fileNameFormat))
{
@SuppressWarnings("unchecked") // instance validation should make this safe!
List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
}
}
if(!StringUtils.hasContent(fileName))
{
//////////////////////////////////
// make default name if missing //
//////////////////////////////////
fileName = table.getLabel() + " " + primaryKey + " " + field.getLabel();
if(StringUtils.hasContent(defaultExtension))
{
//////////////////////////////////////////
// add default extension if we have one //
//////////////////////////////////////////
fileName += "." + defaultExtension;
}
}
record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
record.setDisplayValue(field.getName(), fileName);
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record)
{
boolean fieldHasValue = false;
try
{
if(record.getValue(field.getName()) != null)
{
fieldHasValue = true;
}
else if(field.getIsHeavy())
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
if(heavyFieldLengths != null)
{
Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName()));
if(fieldLength != null && fieldLength > 0)
{
fieldHasValue = true;
}
}
}
}
catch(Exception e)
{
LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record));
}
return fieldHasValue;
}
/*******************************************************************************
**
*******************************************************************************/
@ -1199,7 +1075,7 @@ public class QJavalinImplementation
QueryAction queryAction = new QueryAction();
QueryOutput queryOutput = queryAction.execute(queryInput);
setBlobValuesToDownloadUrls(QContext.getQInstance().getTable(table), queryOutput.getRecords());
QValueFormatter.setBlobValuesToDownloadUrls(QContext.getQInstance().getTable(table), queryOutput.getRecords());
QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS), logPairIfSlow("joins", queryJoins, SLOW_LOG_THRESHOLD_MS));
context.result(JsonUtils.toJson(queryOutput));