mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 13:10:44 +00:00
more support for blobs
- fetch heavy flag for process extract - ignore blobs in bulk edit, load - set download urls in child-list render and javalin via shared method in QValueFormatter also, make ETL load step aware of transform step, and more flexible process summary between them
This commit is contained in:
@ -33,6 +33,7 @@ import java.util.Set;
|
|||||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||||
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
||||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||||
|
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
|
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||||
@ -164,6 +165,8 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
|
|||||||
String joinName = input.getQueryParams().get("joinName");
|
String joinName = input.getQueryParams().get("joinName");
|
||||||
QJoinMetaData join = input.getInstance().getJoin(joinName);
|
QJoinMetaData join = input.getInstance().getJoin(joinName);
|
||||||
String id = input.getQueryParams().get("id");
|
String id = input.getQueryParams().get("id");
|
||||||
|
QTableMetaData leftTable = input.getInstance().getTable(join.getLeftTable());
|
||||||
|
QTableMetaData rightTable = input.getInstance().getTable(join.getRightTable());
|
||||||
|
|
||||||
Integer maxRows = null;
|
Integer maxRows = null;
|
||||||
if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
|
if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
|
||||||
@ -187,8 +190,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
|
|||||||
|
|
||||||
if(record == null)
|
if(record == null)
|
||||||
{
|
{
|
||||||
QTableMetaData table = input.getInstance().getTable(join.getLeftTable());
|
throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
|
||||||
throw (new QNotFoundException("Could not find " + (table == null ? "" : table.getLabel()) + " with primary key " + id));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////
|
||||||
@ -209,6 +211,8 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
|
|||||||
queryInput.setFilter(filter);
|
queryInput.setFilter(filter);
|
||||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||||
|
|
||||||
|
QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
|
||||||
|
|
||||||
int totalRows = queryOutput.getRecords().size();
|
int totalRows = queryOutput.getRecords().size();
|
||||||
if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
|
if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
|
||||||
{
|
{
|
||||||
@ -222,11 +226,10 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
|
|||||||
totalRows = new CountAction().execute(countInput).getCount();
|
totalRows = new CountAction().execute(countInput).getCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
QTableMetaData table = input.getInstance().getTable(join.getRightTable());
|
String tablePath = input.getInstance().getTablePath(rightTable.getName());
|
||||||
String tablePath = input.getInstance().getTablePath(table.getName());
|
|
||||||
String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
|
String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
|
||||||
|
|
||||||
ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, table, tablePath, viewAllLink, totalRows);
|
ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
|
||||||
|
|
||||||
if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
|
if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
|
||||||
{
|
{
|
||||||
|
@ -29,13 +29,18 @@ import java.time.LocalTime;
|
|||||||
import java.time.ZonedDateTime;
|
import java.time.ZonedDateTime;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.Optional;
|
||||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||||
@ -434,4 +439,127 @@ public class QValueFormatter
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** For any BLOB type fields in the list of records, change their value to
|
||||||
|
** the URL where they can be downloaded, and set their display value to a file name.
|
||||||
|
*******************************************************************************/
|
||||||
|
public static void setBlobValuesToDownloadUrls(QTableMetaData table, List<QRecord> records)
|
||||||
|
{
|
||||||
|
for(QFieldMetaData field : table.getFields().values())
|
||||||
|
{
|
||||||
|
if(field.getType().equals(QFieldType.BLOB))
|
||||||
|
{
|
||||||
|
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// file name comes from: //
|
||||||
|
// if there's a FILE_DOWNLOAD adornment, with a FILE_NAME_FIELD value, then the full filename comes from that field //
|
||||||
|
// - unless it was empty - then we do the "default thing": //
|
||||||
|
// else - the "default thing" is: //
|
||||||
|
// - tableLabel primaryKey fieldLabel //
|
||||||
|
// - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot) //
|
||||||
|
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
Optional<FieldAdornment> fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
|
||||||
|
Map<String, Serializable> adornmentValues = Collections.emptyMap();
|
||||||
|
|
||||||
|
if(fileDownloadAdornment.isPresent())
|
||||||
|
{
|
||||||
|
adornmentValues = fileDownloadAdornment.get().getValues();
|
||||||
|
}
|
||||||
|
|
||||||
|
String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
|
||||||
|
String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
|
||||||
|
String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
|
||||||
|
|
||||||
|
for(QRecord record : records)
|
||||||
|
{
|
||||||
|
if(!doesFieldHaveValue(field, record))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
|
||||||
|
String fileName = null;
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////
|
||||||
|
// try to make file name from the fileNameField //
|
||||||
|
//////////////////////////////////////////////////
|
||||||
|
if(StringUtils.hasContent(fileNameField))
|
||||||
|
{
|
||||||
|
fileName = record.getValueString(fileNameField);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!StringUtils.hasContent(fileName))
|
||||||
|
{
|
||||||
|
if(StringUtils.hasContent(fileNameFormat))
|
||||||
|
{
|
||||||
|
@SuppressWarnings("unchecked") // instance validation should make this safe!
|
||||||
|
List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
|
||||||
|
List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
|
||||||
|
fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!StringUtils.hasContent(fileName))
|
||||||
|
{
|
||||||
|
//////////////////////////////////
|
||||||
|
// make default name if missing //
|
||||||
|
//////////////////////////////////
|
||||||
|
fileName = table.getLabel() + " " + primaryKey + " " + field.getLabel();
|
||||||
|
|
||||||
|
if(StringUtils.hasContent(defaultExtension))
|
||||||
|
{
|
||||||
|
//////////////////////////////////////////
|
||||||
|
// add default extension if we have one //
|
||||||
|
//////////////////////////////////////////
|
||||||
|
fileName += "." + defaultExtension;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
|
||||||
|
record.setDisplayValue(field.getName(), fileName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record)
|
||||||
|
{
|
||||||
|
boolean fieldHasValue = false;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if(record.getValue(field.getName()) != null)
|
||||||
|
{
|
||||||
|
fieldHasValue = true;
|
||||||
|
}
|
||||||
|
else if(field.getIsHeavy())
|
||||||
|
{
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
|
||||||
|
if(heavyFieldLengths != null)
|
||||||
|
{
|
||||||
|
Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName()));
|
||||||
|
if(fieldLength != null && fieldLength > 0)
|
||||||
|
{
|
||||||
|
fieldHasValue = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch(Exception e)
|
||||||
|
{
|
||||||
|
LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record));
|
||||||
|
}
|
||||||
|
|
||||||
|
return fieldHasValue;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -716,7 +716,7 @@ public class QInstanceEnricher
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
QFieldMetaData field = table.getField(fieldName);
|
QFieldMetaData field = table.getField(fieldName);
|
||||||
if(field.getIsEditable())
|
if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB))
|
||||||
{
|
{
|
||||||
editableFields.add(field);
|
editableFields.add(field);
|
||||||
}
|
}
|
||||||
@ -774,6 +774,7 @@ public class QInstanceEnricher
|
|||||||
|
|
||||||
List<QFieldMetaData> editableFields = table.getFields().values().stream()
|
List<QFieldMetaData> editableFields = table.getFields().values().stream()
|
||||||
.filter(QFieldMetaData::getIsEditable)
|
.filter(QFieldMetaData::getIsEditable)
|
||||||
|
.filter(f -> !f.getType().equals(QFieldType.BLOB))
|
||||||
.toList();
|
.toList();
|
||||||
|
|
||||||
QFrontendStepMetaData editScreen = new QFrontendStepMetaData()
|
QFrontendStepMetaData editScreen = new QFrontendStepMetaData()
|
||||||
|
@ -29,6 +29,7 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
|
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
|
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
|
||||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||||
|
|
||||||
@ -290,4 +291,25 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
|
|||||||
return (this);
|
return (this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public void addAuditSingleInput(AuditSingleInput auditSingleInput)
|
||||||
|
{
|
||||||
|
if(getAuditInputList() == null)
|
||||||
|
{
|
||||||
|
setAuditInputList(new ArrayList<>());
|
||||||
|
}
|
||||||
|
|
||||||
|
if(getAuditInputList().isEmpty())
|
||||||
|
{
|
||||||
|
getAuditInputList().add(new AuditInput());
|
||||||
|
}
|
||||||
|
|
||||||
|
AuditInput auditInput = getAuditInputList().get(0);
|
||||||
|
auditInput.addAuditSingleInput(auditSingleInput);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -70,6 +70,17 @@ public class AbstractProcessMetaDataBuilder
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public AbstractProcessMetaDataBuilder withInputFieldDefaultValue(String fieldName, Serializable value)
|
||||||
|
{
|
||||||
|
setInputFieldDefaultValue(fieldName, value);
|
||||||
|
return (this);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
|
@ -47,6 +47,8 @@ public abstract class AbstractLoadStep implements BackendStep
|
|||||||
private Optional<QBackendTransaction> transaction = Optional.empty();
|
private Optional<QBackendTransaction> transaction = Optional.empty();
|
||||||
protected QSession session;
|
protected QSession session;
|
||||||
|
|
||||||
|
private AbstractTransformStep transformStep;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
@ -122,4 +124,35 @@ public abstract class AbstractLoadStep implements BackendStep
|
|||||||
return (null);
|
return (null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for transformStep
|
||||||
|
*******************************************************************************/
|
||||||
|
public AbstractTransformStep getTransformStep()
|
||||||
|
{
|
||||||
|
return (this.transformStep);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for transformStep
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setTransformStep(AbstractTransformStep transformStep)
|
||||||
|
{
|
||||||
|
this.transformStep = transformStep;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Fluent setter for transformStep
|
||||||
|
*******************************************************************************/
|
||||||
|
public AbstractLoadStep withTransformStep(AbstractTransformStep transformStep)
|
||||||
|
{
|
||||||
|
this.transformStep = transformStep;
|
||||||
|
return (this);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -109,6 +109,11 @@ public class ExtractViaQueryStep extends AbstractExtractStep
|
|||||||
queryInput.setRecordPipe(getRecordPipe());
|
queryInput.setRecordPipe(getRecordPipe());
|
||||||
queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
|
queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
|
||||||
|
|
||||||
|
if(runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_FETCH_HEAVY_FIELDS))
|
||||||
|
{
|
||||||
|
queryInput.setShouldFetchHeavyFields(true);
|
||||||
|
}
|
||||||
|
|
||||||
customizeInputPreQuery(queryInput);
|
customizeInputPreQuery(queryInput);
|
||||||
|
|
||||||
new QueryAction().execute(queryInput);
|
new QueryAction().execute(queryInput);
|
||||||
|
@ -73,6 +73,8 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
|
|||||||
AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
|
AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
|
||||||
AbstractLoadStep loadStep = getLoadStep(runBackendStepInput);
|
AbstractLoadStep loadStep = getLoadStep(runBackendStepInput);
|
||||||
|
|
||||||
|
loadStep.setTransformStep(transformStep);
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////
|
||||||
// let the load step override the capacity for the record pipe. //
|
// let the load step override the capacity for the record pipe. //
|
||||||
// this is useful for slower load steps - so that the extract step doesn't //
|
// this is useful for slower load steps - so that the extract step doesn't //
|
||||||
@ -140,13 +142,17 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
|
|||||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
// get the process summary from the load step, if it's a summary-provider -- else, use the transform step (which is always a provider) //
|
// get the process summary from the load step, if it's a summary-provider -- else, use the transform step (which is always a provider) //
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
ArrayList<ProcessSummaryLineInterface> processSummaryLines = null;
|
||||||
if(loadStep instanceof ProcessSummaryProviderInterface provider)
|
if(loadStep instanceof ProcessSummaryProviderInterface provider)
|
||||||
{
|
{
|
||||||
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, provider.doGetProcessSummary(runBackendStepOutput, true));
|
processSummaryLines = provider.doGetProcessSummary(runBackendStepOutput, true);
|
||||||
|
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryLines);
|
||||||
}
|
}
|
||||||
else
|
|
||||||
|
if(CollectionUtils.nullSafeIsEmpty(processSummaryLines))
|
||||||
{
|
{
|
||||||
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, transformStep.doGetProcessSummary(runBackendStepOutput, true));
|
processSummaryLines = transformStep.doGetProcessSummary(runBackendStepOutput, true);
|
||||||
|
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryLines);
|
||||||
}
|
}
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
@ -81,6 +81,7 @@ public class StreamedETLWithFrontendProcess
|
|||||||
public static final String FIELD_DESTINATION_TABLE = "destinationTable"; // String
|
public static final String FIELD_DESTINATION_TABLE = "destinationTable"; // String
|
||||||
public static final String FIELD_RECORD_COUNT = "recordCount"; // Integer
|
public static final String FIELD_RECORD_COUNT = "recordCount"; // Integer
|
||||||
public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter)
|
public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter)
|
||||||
|
public static final String FIELD_FETCH_HEAVY_FIELDS = "fetchHeavyFields"; // Boolean
|
||||||
|
|
||||||
public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean
|
public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean
|
||||||
public static final String FIELD_DO_FULL_VALIDATION = "doFullValidation"; // Boolean
|
public static final String FIELD_DO_FULL_VALIDATION = "doFullValidation"; // Boolean
|
||||||
@ -142,6 +143,7 @@ public class StreamedETLWithFrontendProcess
|
|||||||
.withCode(new QCodeReference(StreamedETLPreviewStep.class))
|
.withCode(new QCodeReference(StreamedETLPreviewStep.class))
|
||||||
.withInputData(new QFunctionInputMetaData()
|
.withInputData(new QFunctionInputMetaData()
|
||||||
.withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE)))
|
.withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE)))
|
||||||
|
.withField(new QFieldMetaData(FIELD_FETCH_HEAVY_FIELDS, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_FETCH_HEAVY_FIELDS, false)))
|
||||||
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE)))
|
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE)))
|
||||||
.withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, true)))
|
.withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, true)))
|
||||||
.withField(new QFieldMetaData(FIELD_DO_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.get(FIELD_DO_FULL_VALIDATION)))
|
.withField(new QFieldMetaData(FIELD_DO_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.get(FIELD_DO_FULL_VALIDATION)))
|
||||||
|
@ -32,7 +32,6 @@ import java.time.Duration;
|
|||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
@ -858,7 +857,7 @@ public class QJavalinImplementation
|
|||||||
+ table.getFields().get(table.getPrimaryKeyField()).getLabel() + " of " + primaryKey));
|
+ table.getFields().get(table.getPrimaryKeyField()).getLabel() + " of " + primaryKey));
|
||||||
}
|
}
|
||||||
|
|
||||||
setBlobValuesToDownloadUrls(table, List.of(record));
|
QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record));
|
||||||
|
|
||||||
QJavalinAccessLogger.logEndSuccess();
|
QJavalinAccessLogger.logEndSuccess();
|
||||||
context.result(JsonUtils.toJson(record));
|
context.result(JsonUtils.toJson(record));
|
||||||
@ -872,129 +871,6 @@ public class QJavalinImplementation
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
|
||||||
** For any BLOB type fields in the list of records, change their value to
|
|
||||||
** the URL where they can be downloaded, and set their display value to a file name.
|
|
||||||
*******************************************************************************/
|
|
||||||
private static void setBlobValuesToDownloadUrls(QTableMetaData table, List<QRecord> records)
|
|
||||||
{
|
|
||||||
for(QFieldMetaData field : table.getFields().values())
|
|
||||||
{
|
|
||||||
if(field.getType().equals(QFieldType.BLOB))
|
|
||||||
{
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// file name comes from: //
|
|
||||||
// if there's a FILE_DOWNLOAD adornment, with a FILE_NAME_FIELD value, then the full filename comes from that field //
|
|
||||||
// - unless it was empty - then we do the "default thing": //
|
|
||||||
// else - the "default thing" is: //
|
|
||||||
// - tableLabel primaryKey fieldLabel //
|
|
||||||
// - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot) //
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
Optional<FieldAdornment> fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
|
|
||||||
Map<String, Serializable> adornmentValues = Collections.emptyMap();
|
|
||||||
|
|
||||||
if(fileDownloadAdornment.isPresent())
|
|
||||||
{
|
|
||||||
adornmentValues = fileDownloadAdornment.get().getValues();
|
|
||||||
}
|
|
||||||
|
|
||||||
String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
|
|
||||||
String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
|
|
||||||
String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
|
|
||||||
|
|
||||||
for(QRecord record : records)
|
|
||||||
{
|
|
||||||
if(!doesFieldHaveValue(field, record))
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
|
|
||||||
String fileName = null;
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
// try to make file name from the fileNameField //
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
if(StringUtils.hasContent(fileNameField))
|
|
||||||
{
|
|
||||||
fileName = record.getValueString(fileNameField);
|
|
||||||
}
|
|
||||||
|
|
||||||
if(!StringUtils.hasContent(fileName))
|
|
||||||
{
|
|
||||||
if(StringUtils.hasContent(fileNameFormat))
|
|
||||||
{
|
|
||||||
@SuppressWarnings("unchecked") // instance validation should make this safe!
|
|
||||||
List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
|
|
||||||
List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
|
|
||||||
fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if(!StringUtils.hasContent(fileName))
|
|
||||||
{
|
|
||||||
//////////////////////////////////
|
|
||||||
// make default name if missing //
|
|
||||||
//////////////////////////////////
|
|
||||||
fileName = table.getLabel() + " " + primaryKey + " " + field.getLabel();
|
|
||||||
|
|
||||||
if(StringUtils.hasContent(defaultExtension))
|
|
||||||
{
|
|
||||||
//////////////////////////////////////////
|
|
||||||
// add default extension if we have one //
|
|
||||||
//////////////////////////////////////////
|
|
||||||
fileName += "." + defaultExtension;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
|
|
||||||
record.setDisplayValue(field.getName(), fileName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
|
||||||
**
|
|
||||||
*******************************************************************************/
|
|
||||||
private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record)
|
|
||||||
{
|
|
||||||
boolean fieldHasValue = false;
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if(record.getValue(field.getName()) != null)
|
|
||||||
{
|
|
||||||
fieldHasValue = true;
|
|
||||||
}
|
|
||||||
else if(field.getIsHeavy())
|
|
||||||
{
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
|
|
||||||
if(heavyFieldLengths != null)
|
|
||||||
{
|
|
||||||
Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName()));
|
|
||||||
if(fieldLength != null && fieldLength > 0)
|
|
||||||
{
|
|
||||||
fieldHasValue = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch(Exception e)
|
|
||||||
{
|
|
||||||
LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record));
|
|
||||||
}
|
|
||||||
|
|
||||||
return fieldHasValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
@ -1199,7 +1075,7 @@ public class QJavalinImplementation
|
|||||||
QueryAction queryAction = new QueryAction();
|
QueryAction queryAction = new QueryAction();
|
||||||
QueryOutput queryOutput = queryAction.execute(queryInput);
|
QueryOutput queryOutput = queryAction.execute(queryInput);
|
||||||
|
|
||||||
setBlobValuesToDownloadUrls(QContext.getQInstance().getTable(table), queryOutput.getRecords());
|
QValueFormatter.setBlobValuesToDownloadUrls(QContext.getQInstance().getTable(table), queryOutput.getRecords());
|
||||||
|
|
||||||
QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS), logPairIfSlow("joins", queryJoins, SLOW_LOG_THRESHOLD_MS));
|
QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS), logPairIfSlow("joins", queryJoins, SLOW_LOG_THRESHOLD_MS));
|
||||||
context.result(JsonUtils.toJson(queryOutput));
|
context.result(JsonUtils.toJson(queryOutput));
|
||||||
|
Reference in New Issue
Block a user