QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updated interface in sync processes; more status updates in ETL processes; Basepull only update timestamp if ran as basepull; javalin report endpoint;

Browse Source
This commit is contained in:
darin.kelkhoff
2022-12-19 10:00:29 -06:00
parent 1b672afcd0
commit e1c53b9d48
34 changed files with 1655 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java
View File

@ -80,6 +80,7 @@ public class RunProcessAction
// indicator that the timestamp field should be updated - e.g., the execute step is finished. //
////////////////////////////////////////////////////////////////////////////////////////////////
public static final String BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD = "basepullReadyToUpdateTimestamp";
public static final String BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD = "basepullDidQueryUsingTimestamp";
@ -190,11 +191,14 @@ public class RunProcessAction
}
}
////////////////////////////////////////////////////////////////////////////////////
// if 'basepull' style process, update the stored basepull timestamp //
// but only when we've been signaled to do so - i.e., after an Execute step runs. //
////////////////////////////////////////////////////////////////////////////////////
if(basepullConfiguration != null && BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD))))
///////////////////////////////////////////////////////////////////////////
// if 'basepull' style process, update the stored basepull timestamp //
// but only when we've been signaled to do so - i.e., only if we did our //
// query using the timestamp field, and only after an Execute step runs. //
///////////////////////////////////////////////////////////////////////////
if(basepullConfiguration != null
&& BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD)))
&& BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD))))
{
storeLastRunTime(runProcessInput, process, basepullConfiguration);
}

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java
View File

@ -179,6 +179,17 @@ public class GenerateReportAction
}
outputSummaries(reportInput);
reportStreamer.finish();
try
{
reportInput.getReportOutputStream().close();
}
catch(Exception e)
{
throw (new QReportingException("Error completing report", e));
}
}
@ -527,8 +538,6 @@ public class GenerateReportAction
reportStreamer.addTotalsRow(summaryOutput.totalRow);
}
}
reportStreamer.finish();
}

155
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java Normal file
View File

@ -0,0 +1,155 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** JSON export format implementation
*******************************************************************************/
public class JsonExportStreamer implements ExportStreamerInterface
{
private static final Logger LOG = LogManager.getLogger(JsonExportStreamer.class);
private ExportInput exportInput;
private QTableMetaData table;
private List<QFieldMetaData> fields;
private OutputStream outputStream;
private boolean needComma = false;
/*******************************************************************************
**
*******************************************************************************/
public JsonExportStreamer()
{
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
{
this.exportInput = exportInput;
this.fields = fields;
table = exportInput.getTable();
outputStream = this.exportInput.getReportOutputStream();
try
{
outputStream.write("[".getBytes(StandardCharsets.UTF_8));
}
catch(IOException e)
{
throw (new QReportingException("Error starting report output", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void addRecords(List<QRecord> qRecords) throws QReportingException
{
LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
for(QRecord qRecord : qRecords)
{
writeRecord(qRecord);
}
}
/*******************************************************************************
**
*******************************************************************************/
private void writeRecord(QRecord qRecord) throws QReportingException
{
try
{
if(needComma)
{
outputStream.write(",".getBytes(StandardCharsets.UTF_8));
}
String json = JsonUtils.toJson(qRecord);
outputStream.write(json.getBytes(StandardCharsets.UTF_8));
outputStream.flush(); // todo - less often?
needComma = true;
}
catch(Exception e)
{
throw (new QReportingException("Error writing JSON report", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void addTotalsRow(QRecord record) throws QReportingException
{
writeRecord(record);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void finish() throws QReportingException
{
try
{
outputStream.write("]".getBytes(StandardCharsets.UTF_8));
}
catch(IOException e)
{
throw (new QReportingException("Error ending report output", e));
}
}
}

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java
View File

@ -38,7 +38,7 @@ import org.apache.logging.log4j.Logger;
** Base input class for all Q actions.
**
*******************************************************************************/
public abstract class AbstractActionInput
public class AbstractActionInput
{
private static final Logger LOG = LogManager.getLogger(AbstractActionInput.class);
@ -69,6 +69,17 @@ public abstract class AbstractActionInput
/*******************************************************************************
**
*******************************************************************************/
public AbstractActionInput(QInstance instance, QSession session)
{
this(instance);
this.session = session;
}
/*******************************************************************************
** performance instance validation (if not previously done).
*******************************************************************************/

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java
View File

@ -27,6 +27,7 @@ import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.actions.reporting.CsvExportStreamer;
import com.kingsrook.qqq.backend.core.actions.reporting.ExcelExportStreamer;
import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
import com.kingsrook.qqq.backend.core.actions.reporting.JsonExportStreamer;
import com.kingsrook.qqq.backend.core.actions.reporting.ListOfMapsExportStreamer;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -39,6 +40,7 @@ import org.dhatim.fastexcel.Worksheet;
public enum ReportFormat
{
XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelExportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
JSON(null, null, JsonExportStreamer::new, "application/json"),
CSV(null, null, CsvExportStreamer::new, "text/csv"),
LIST_OF_MAPS(null, null, ListOfMapsExportStreamer::new, null);

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.reporting;
import java.io.OutputStream;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -118,6 +119,20 @@ public class ReportInput extends AbstractTableActionInput
/*******************************************************************************
**
*******************************************************************************/
public void addInputValue(String key, Serializable value)
{
if(this.inputValues == null)
{
this.inputValues = new HashMap<>();
}
this.inputValues.put(key, value);
}
/*******************************************************************************
** Getter for filename
**

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java
View File

@ -104,38 +104,7 @@ public class QFrontendTableMetaData
Set<Capability> enabledCapabilities = new HashSet<>();
for(Capability capability : Capability.values())
{
///////////////////////////////////////////////
// by default, every table can do everything //
///////////////////////////////////////////////
boolean hasCapability = true;
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the table's backend says the capability is disabled, then by default, then the capability is disabled... //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(backend.getDisabledCapabilities().contains(capability))
{
hasCapability = false;
/////////////////////////////////////////////////////////////////
// unless the table overrides that and says that it IS enabled //
/////////////////////////////////////////////////////////////////
if(table.getEnabledCapabilities().contains(capability))
{
hasCapability = true;
}
}
else
{
/////////////////////////////////////////////////////////////////////////////////////////
// if the backend doesn't specify the capability, then disable it if the table says so //
/////////////////////////////////////////////////////////////////////////////////////////
if(table.getDisabledCapabilities().contains(capability))
{
hasCapability = false;
}
}
if(hasCapability)
if(table.isCapabilityEnabled(backend, capability))
{
///////////////////////////////////////
// todo - check if user is allowed!! //

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
/*******************************************************************************
@ -65,4 +66,16 @@ public class AbstractProcessMetaDataBuilder
.filter(f -> f.getName().equals(fieldName)).findFirst()
.ifPresent(f -> f.setDefaultValue(value));
}
/*******************************************************************************
**
*******************************************************************************/
public AbstractProcessMetaDataBuilder withBasepullConfiguration(BasepullConfiguration basepullConfiguration)
{
processMetaData.setBasepullConfiguration(basepullConfiguration);
return (this);
}
}

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java
View File

@ -34,8 +34,8 @@ public enum Capability
TABLE_INSERT,
TABLE_UPDATE,
TABLE_DELETE
//////////////////////////////////////////////////////////////////////////
// keep these values in sync with AdornmentType.ts in qqq-frontend-core //
//////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////
// keep these values in sync with Capability.ts in qqq-frontend-core //
///////////////////////////////////////////////////////////////////////
}

35
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java
View File

@ -41,6 +41,7 @@ public class QFieldSection
private QIcon icon;
private boolean isHidden = false;
private Integer gridColumns;
@ -328,4 +329,38 @@ public class QFieldSection
return (this);
}
/*******************************************************************************
** Getter for gridColumns
**
*******************************************************************************/
public Integer getGridColumns()
{
return gridColumns;
}
/*******************************************************************************
** Setter for gridColumns
**
*******************************************************************************/
public void setGridColumns(Integer gridColumns)
{
this.gridColumns = gridColumns;
}
/*******************************************************************************
** Fluent setter for gridColumns
**
*******************************************************************************/
public QFieldSection withGridColumns(Integer gridColumns)
{
this.gridColumns = gridColumns;
return (this);
}
}

41
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java
View File

@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizer;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntityField;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
@ -1037,4 +1038,44 @@ public class QTableMetaData implements QAppChildMetaData, Serializable
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public boolean isCapabilityEnabled(QBackendMetaData backend, Capability capability)
{
///////////////////////////////////////////////
// by default, every table can do everything //
///////////////////////////////////////////////
boolean hasCapability = true;
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the table's backend says the capability is disabled, then by default, then the capability is disabled... //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(backend.getDisabledCapabilities().contains(capability))
{
hasCapability = false;
/////////////////////////////////////////////////////////////////
// unless the table overrides that and says that it IS enabled //
/////////////////////////////////////////////////////////////////
if(getEnabledCapabilities().contains(capability))
{
hasCapability = true;
}
}
else
{
/////////////////////////////////////////////////////////////////////////////////////////
// if the backend doesn't specify the capability, then disable it if the table says so //
/////////////////////////////////////////////////////////////////////////////////////////
if(getDisabledCapabilities().contains(capability))
{
hasCapability = false;
}
}
return (hasCapability);
}
}

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java
View File

@ -51,6 +51,7 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
try
{
queryFilter = super.getQueryFilter(runBackendStepInput);
return (queryFilter);
}
catch(QException qe)
{
@ -77,6 +78,12 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
queryFilter.addOrderBy(new QFilterOrderBy(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD)));
/////////////////////////////////////////////////////////////////////////////////////
// put a flag in the process's values, to note that we did use the timestamp field //
// this will later be checked to see if we should update the timestamp too. //
/////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue(RunProcessAction.BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD, true);
return (queryFilter);
}

102
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java
View File

@ -1,102 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for performing the Load step of a basic ETL process using update.
*******************************************************************************/
public class BasicETLLoadAsUpdateFunction implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLLoadAsUpdateFunction.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
//////////////////////////////////////////////////////
// exit early with no-op if no records made it here //
//////////////////////////////////////////////////////
List<QRecord> inputRecords = runBackendStepInput.getRecords();
LOG.info("Received [" + inputRecords.size() + "] records to load using update");
if(CollectionUtils.nullSafeIsEmpty(inputRecords))
{
runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0);
return;
}
/////////////////////////////////////////////////////////////////
// put the destination table name in all records being updated //
/////////////////////////////////////////////////////////////////
String table = runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
for(QRecord record : inputRecords)
{
record.setTableName(table);
}
//////////////////////////////////////////
// run an update request on the records //
//////////////////////////////////////////
int recordsUpdated = 0;
List<QRecord> outputRecords = new ArrayList<>();
int pageSize = 1000; // todo - make this a field?
for(List<QRecord> page : CollectionUtils.getPages(inputRecords, pageSize))
{
LOG.info("Updating a page of [" + page.size() + "] records. Progress: " + recordsUpdated + " loaded out of " + inputRecords.size() + " total");
runBackendStepInput.getAsyncJobCallback().updateStatus("Updating records", recordsUpdated, inputRecords.size());
UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance());
updateInput.setSession(runBackendStepInput.getSession());
updateInput.setTableName(table);
updateInput.setRecords(page);
UpdateAction updateAction = new UpdateAction();
UpdateOutput updateResult = updateAction.execute(updateInput);
outputRecords.addAll(updateResult.getRecords());
recordsUpdated += updateResult.getRecords().size();
}
runBackendStepOutput.setRecords(outputRecords);
runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsUpdated);
}
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
View File

@ -62,6 +62,8 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
try
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Executing Process");
///////////////////////////////////////////////////////
// set up the extract, transform, and load functions //
///////////////////////////////////////////////////////
@ -137,6 +139,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
//////////////////////////////////////////////////////////////////////////////
// set the flag to state that the basepull timestamp should be updated now. //
// (upstream will check if the process was running as a basepull) //
//////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue(RunProcessAction.BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD, true);

30
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java
View File

@ -32,7 +32,11 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -54,6 +58,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Integer limit = PROCESS_OUTPUT_RECORD_LIST_LIMIT; // todo - use a field instead of hard-coded here?
runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Preview");
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the do-full-validation flag has already been set, then do the validation step instead of this one //
@ -69,7 +74,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
if(runBackendStepInput.getFrontendStepBehavior() != null && runBackendStepInput.getFrontendStepBehavior().equals(RunProcessInput.FrontendStepBehavior.SKIP))
{
LOG.debug("Skipping preview because frontent behavior is [" + RunProcessInput.FrontendStepBehavior.SKIP + "].");
LOG.debug("Skipping preview because frontend behavior is [" + RunProcessInput.FrontendStepBehavior.SKIP + "].");
return;
}
@ -91,8 +96,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
extractStep.setRecordPipe(recordPipe);
extractStep.preRun(runBackendStepInput, runBackendStepOutput);
Integer recordCount = extractStep.doCount(runBackendStepInput);
runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount);
countRecords(runBackendStepInput, runBackendStepOutput, extractStep);
AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
transformStep.preRun(runBackendStepInput, runBackendStepOutput);
@ -126,6 +130,26 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
/*******************************************************************************
**
*******************************************************************************/
private void countRecords(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, AbstractExtractStep extractStep) throws QException
{
String sourceTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(sourceTableName);
if(StringUtils.hasContent(sourceTableName))
{
QBackendMetaData sourceTableBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTableName);
if(sourceTable.isCapabilityEnabled(sourceTableBackend, Capability.TABLE_COUNT))
{
Integer recordCount = extractStep.doCount(runBackendStepInput);
runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount);
}
}
}
/*******************************************************************************
**
*******************************************************************************/

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java
View File

@ -81,6 +81,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back
//////////////////////////////////////////////////////////
// basically repeat the preview step, but with no limit //
//////////////////////////////////////////////////////////
runBackendStepInput.getAsyncJobCallback().updateStatus("Validating Records");
RecordPipe recordPipe = new RecordPipe();
AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
extractStep.setLimit(null);

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java
View File

@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
/*******************************************************************************
@ -389,5 +390,17 @@ public class StreamedETLWithFrontendProcess
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Builder withBasepullConfiguration(BasepullConfiguration basepullConfiguration)
{
processMetaData.setBasepullConfiguration(basepullConfiguration);
return (this);
}
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java
View File

@ -65,6 +65,7 @@ public class MockBackendStep implements BackendStep
runBackendStepOutput.setValues(runBackendStepInput.getValues());
runBackendStepOutput.addValue(FIELD_MOCK_VALUE, MOCK_VALUE);
runBackendStepOutput.addValue("noOfPeopleGreeted", runBackendStepInput.getRecords().size());
runBackendStepOutput.addValue(RunProcessAction.BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD, true);
runBackendStepOutput.addValue(RunProcessAction.BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD, true);
if("there".equalsIgnoreCase(runBackendStepInput.getValueString(FIELD_GREETING_SUFFIX)))

163
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java
View File

@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -45,11 +44,15 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer;
import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
import com.kingsrook.qqq.backend.core.processes.utils.RecordLookupHelper;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
@ -69,44 +72,11 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
.withSingularPastMessage("was not synced, because it is ")
.withPluralPastMessage("were not synced, because they are ");
private RunBackendStepInput runBackendStepInput = null;
protected RunBackendStepInput runBackendStepInput = null;
protected RecordLookupHelper recordLookupHelper = null;
private QPossibleValueTranslator possibleValueTranslator;
private Map<String, Map<Serializable, QRecord>> tableMaps = new HashMap<>();
/*******************************************************************************
**
*******************************************************************************/
protected QRecord getRecord(String tableName, String fieldName, Serializable value) throws QException
{
if(!tableMaps.containsKey(tableName))
{
Map<Serializable, QRecord> recordMap = GeneralProcessUtils.loadTableToMap(runBackendStepInput, tableName, fieldName);
tableMaps.put(tableName, recordMap);
}
return (tableMaps.get(tableName).get(value));
}
/*******************************************************************************
**
*******************************************************************************/
protected Serializable getRecordField(String tableName, String fieldName, Serializable value, String outputField) throws QException
{
QRecord record = getRecord(tableName, fieldName, value);
if(record == null)
{
return (null);
}
return (record.getValue(outputField));
}
/*******************************************************************************
@ -121,12 +91,65 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
/*******************************************************************************
** Map a record from the source table to the destination table. e.g., put
** values into the destinationRecord, from the sourceRecord.
**
** The destinationRecord will already be constructed, and will actually already
** be the record being updated, in the case of an update. It'll be empty (newly
** constructed) for an insert.
*******************************************************************************/
public abstract QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException;
/*******************************************************************************
** Specify a list of tableName/keyColumnName pairs to run through
** the preloadRecords method of the recordLookupHelper.
*******************************************************************************/
protected List<Pair<String, String>> getLookupsToPreLoad()
{
return (null);
}
/*******************************************************************************
** Define the query filter to find existing records. e.g., for determining
** insert vs. update. Subclasses may override this to customize the behavior,
** e.g., in case an additional field is needed in the query.
*******************************************************************************/
protected QQueryFilter getExistingRecordQueryFilter(RunBackendStepInput runBackendStepInput, List<Serializable> sourceKeyList)
{
String destinationTableForeignKeyField = getSyncProcessConfig().destinationTableForeignKey;
return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList));
}
/*******************************************************************************
** Define the config for this process - e.g., what fields & tables are used.
*******************************************************************************/
protected abstract SyncProcessConfig getSyncProcessConfig();
/*******************************************************************************
** Record to store the config for this process - e.g., what fields & tables are used.
*******************************************************************************/
public record SyncProcessConfig(String sourceTable, String sourceTableKeyField, String destinationTable, String destinationTableForeignKey)
{
/*******************************************************************************
** artificial method, here to make jacoco see that this class is indeed
** included in test coverage...
*******************************************************************************/
void noop()
{
System.out.println("noop");
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -139,9 +162,33 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
}
this.runBackendStepInput = runBackendStepInput;
String sourceTableKeyField = runBackendStepInput.getValueString(TableSyncProcess.FIELD_SOURCE_TABLE_KEY_FIELD);
String destinationTableForeignKeyField = runBackendStepInput.getValueString(TableSyncProcess.FIELD_DESTINATION_TABLE_FOREIGN_KEY);
String destinationTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE);
if(this.recordLookupHelper == null)
{
initializeRecordLookupHelper(runBackendStepInput);
}
SyncProcessConfig config = getSyncProcessConfig();
String sourceTableKeyField = config.sourceTableKeyField;
String destinationTableForeignKeyField = config.destinationTableForeignKey;
String destinationTableName = config.destinationTable;
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, destinationTableName);
if(!StringUtils.hasContent(sourceTableKeyField))
{
throw (new IllegalStateException("Missing sourceTableKeyField in config for " + getClass().getSimpleName()));
}
if(!StringUtils.hasContent(destinationTableForeignKeyField))
{
throw (new IllegalStateException("Missing destinationTableForeignKey in config for " + getClass().getSimpleName()));
}
if(!StringUtils.hasContent(destinationTableName))
{
throw (new IllegalStateException("Missing destinationTable in config for " + getClass().getSimpleName()));
}
//////////////////////////////////////
// extract keys from source records //
@ -161,9 +208,8 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance());
queryInput.setSession(runBackendStepInput.getSession());
queryInput.setTableName(destinationTableName);
queryInput.setFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList))
);
QQueryFilter filter = getExistingRecordQueryFilter(runBackendStepInput, sourceKeyList);
queryInput.setFilter(filter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
existingRecordsByForeignKey = CollectionUtils.recordsToMap(queryOutput.getRecords(), destinationTableForeignKeyField);
}
@ -171,10 +217,10 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
/////////////////////////////////////////////////////////////////
// foreach source record, build the record we'll insert/update //
/////////////////////////////////////////////////////////////////
QFieldMetaData destinationForeignKeyField = runBackendStepInput.getInstance().getTable(destinationTableName).getField(destinationTableForeignKeyField);
for(QRecord sourceRecord : runBackendStepInput.getRecords())
{
Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField);
QRecord existingRecord = existingRecordsByForeignKey.get(sourceKeyValue);
if(sourceKeyValue == null || "".equals(sourceKeyValue))
{
@ -194,6 +240,14 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
continue;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// look for the existing record - note - we may need to type-convert here, the sourceKey value //
// from the source table to the destinationKey. e.g., if source table had an integer, and the //
// destination has a string. //
/////////////////////////////////////////////////////////////////////////////////////////////////
Serializable sourceKeyValueInTargetFieldType = ValueUtils.getValueAsFieldType(destinationForeignKeyField.getType(), sourceKeyValue);
QRecord existingRecord = existingRecordsByForeignKey.get(sourceKeyValueInTargetFieldType);
QRecord recordToStore;
if(existingRecord != null)
{
@ -227,4 +281,25 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
}
}
/*******************************************************************************
** If needed, init a record lookup helper for this process.
*******************************************************************************/
protected void initializeRecordLookupHelper(RunBackendStepInput runBackendStepInput) throws QException
{
this.recordLookupHelper = new RecordLookupHelper(runBackendStepInput);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if there's only 1 record, don't bother preloading all records - just do the single lookup by the single needed key. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getRecords().size() > 1)
{
for(Pair<String, String> pair : CollectionUtils.nonNullList(getLookupsToPreLoad()))
{
recordLookupHelper.preloadRecords(pair.getA(), pair.getB());
}
}
}
}

76
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java
View File

@ -24,12 +24,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
import java.util.Collections;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertOrUpdateStep;
@ -84,60 +86,28 @@ public class TableSyncProcess
/*******************************************************************************
** Fluent setter for sourceTableKeyField
**
*******************************************************************************/
public Builder withSourceTableKeyField(String sourceTableKeyField)
{
setInputFieldDefaultValue(FIELD_SOURCE_TABLE_KEY_FIELD, sourceTableKeyField);
return (this);
}
/*******************************************************************************
** Fluent setter for destinationTableForeignKeyField
**
*******************************************************************************/
public Builder withDestinationTableForeignKeyField(String destinationTableForeignKeyField)
{
setInputFieldDefaultValue(FIELD_DESTINATION_TABLE_FOREIGN_KEY, destinationTableForeignKeyField);
return (this);
}
/*******************************************************************************
** Fluent setter for transformStepClass
** Fluent setter for transformStepClass. Note - call this method also makes
** sourceTable and destinationTable be set - by getting them from the
** SyncProcessConfig record defined in the step class.
**
*******************************************************************************/
public Builder withSyncTransformStepClass(Class<? extends AbstractTableSyncTransformStep> transformStepClass)
{
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE, new QCodeReference(transformStepClass));
return (this);
}
AbstractTableSyncTransformStep.SyncProcessConfig config;
try
{
AbstractTableSyncTransformStep transformStep = transformStepClass.getConstructor().newInstance();
config = transformStep.getSyncProcessConfig();
}
catch(Exception e)
{
throw (new QRuntimeException("Error setting up process with transform step class: " + transformStepClass.getName(), e));
}
/*******************************************************************************
** Fluent setter for sourceTable
**
*******************************************************************************/
public Builder withSourceTable(String sourceTable)
{
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, sourceTable);
return (this);
}
/*******************************************************************************
** Fluent setter for destinationTable
**
*******************************************************************************/
public Builder withDestinationTable(String destinationTable)
{
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, destinationTable);
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, config.sourceTable());
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, config.destinationTable());
return (this);
}
@ -204,5 +174,17 @@ public class TableSyncProcess
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public StreamedETLWithFrontendProcess.Builder withBasepullConfiguration(BasepullConfiguration basepullConfiguration)
{
processMetaData.setBasepullConfiguration(basepullConfiguration);
return (this);
}
}
}

23
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java
View File

@ -203,10 +203,15 @@ public class GeneralProcessUtils
*******************************************************************************/
public static Optional<QRecord> getRecordByField(AbstractActionInput parentActionInput, String tableName, String fieldName, Serializable fieldValue) throws QException
{
if(fieldValue == null)
{
return (Optional.empty());
}
QueryInput queryInput = new QueryInput(parentActionInput.getInstance());
queryInput.setSession(parentActionInput.getSession());
queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, List.of(fieldValue))));
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue)));
queryInput.setLimit(1);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
return (queryOutput.getRecords().stream().findFirst());
@ -321,10 +326,26 @@ public class GeneralProcessUtils
** too many rows... Caveat emptor.
*******************************************************************************/
public static Map<Serializable, QRecord> loadTableToMap(AbstractActionInput parentActionInput, String tableName, String keyFieldName) throws QException
{
return (loadTableToMap(parentActionInput, tableName, keyFieldName, (QQueryFilter) null));
}
/*******************************************************************************
** Load rows from a table matching the specified filter, into a map, keyed by the keyFieldName.
**
** Note - null values from the key field are NOT put in the map.
**
** If multiple values are found for the key, they'll squash each other, and only
** one (random) value will appear.
*******************************************************************************/
public static Map<Serializable, QRecord> loadTableToMap(AbstractActionInput parentActionInput, String tableName, String keyFieldName, QQueryFilter filter) throws QException
{
QueryInput queryInput = new QueryInput(parentActionInput.getInstance());
queryInput.setSession(parentActionInput.getSession());
queryInput.setTableName(tableName);
queryInput.setFilter(filter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
List<QRecord> records = queryOutput.getRecords();

153
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java Normal file
View File

@ -0,0 +1,153 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.utils;
import java.io.Serializable;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** Utility to help processes lookup records. Caches lookups - and potentially
** can pre-load entire tables or subsets of tables.
**
*******************************************************************************/
public class RecordLookupHelper
{
private final AbstractActionInput actionInput;
private Map<String, Map<Serializable, QRecord>> foreignRecordMaps = new HashMap<>();
private Set<String> preloadedKeys = new HashSet<>();
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public RecordLookupHelper(AbstractActionInput actionInput)
{
this.actionInput = actionInput;
}
/*******************************************************************************
** Fetch a record from a table by a key field (doesn't have to be its primary key).
*******************************************************************************/
public QRecord getRecordByKey(String tableName, String keyFieldName, Serializable key) throws QException
{
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> recordMap = foreignRecordMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
if(!recordMap.containsKey(key))
{
Optional<QRecord> optRecord = GeneralProcessUtils.getRecordByField(actionInput, tableName, keyFieldName, key);
recordMap.put(key, optRecord.orElse(null));
}
return (recordMap.get(key));
}
/*******************************************************************************
** Optimization - to pre-load the records in a single query, which would otherwise
** have to be looked up one-by-one.
*******************************************************************************/
public void preloadRecords(String tableName, String keyFieldName) throws QException
{
String mapKey = tableName + "." + keyFieldName;
if(!preloadedKeys.contains(mapKey))
{
Map<Serializable, QRecord> recordMap = GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName);
foreignRecordMaps.put(mapKey, recordMap);
preloadedKeys.add(mapKey);
}
}
/*******************************************************************************
** Get a value from a record, by doing a lookup on the specified keyFieldName,
** for the specified key value.
**
*******************************************************************************/
public Serializable getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key) throws QException
{
QRecord record = getRecordByKey(tableName, keyFieldName, key);
if(record == null)
{
return (null);
}
return (record.getValue(requestedField));
}
/*******************************************************************************
** Get a value from a record, in the requested type, by doing a lookup on the
** specified keyFieldName, for the specified key value.
**
*******************************************************************************/
public <T extends Serializable> T getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key, Class<T> type) throws QException
{
Serializable value = getRecordValue(tableName, requestedField, keyFieldName, key);
return (ValueUtils.getValueAsType(type, value));
}
/*******************************************************************************
** Get the id (primary key) value from a record, by doing a lookup on the
** specified keyFieldName, for the specified key value.
**
*******************************************************************************/
public Serializable getRecordId(String tableName, String keyFieldName, Serializable key) throws QException
{
String primaryKeyField = actionInput.getInstance().getTable(tableName).getPrimaryKeyField();
return (getRecordValue(tableName, primaryKeyField, keyFieldName, key));
}
/*******************************************************************************
** Get the id (primary key) value from a record, in the requested type, by doing
** a lookup on the specified keyFieldName, for the specified key value.
**
*******************************************************************************/
public <T extends Serializable> T getRecordId(String tableName, String keyFieldName, Serializable key, Class<T> type) throws QException
{
Serializable value = getRecordId(tableName, keyFieldName, key);
return (ValueUtils.getValueAsType(type, value));
}
}

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/Pair.java
View File

@ -46,6 +46,16 @@ public class Pair<A, B> implements Cloneable
/*******************************************************************************
** static constructor (factory)
*******************************************************************************/
public static <A, B> Pair<A, B> of(A a, B b)
{
return (new Pair<>(a, b));
}
/*******************************************************************************
**
*******************************************************************************/

5
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java
View File

@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -53,7 +54,8 @@ public class QPossibleValueTranslatorTest
**
*******************************************************************************/
@BeforeEach
void beforeEach()
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
MemoryRecordStore.resetStatistics();
@ -164,6 +166,7 @@ public class QPossibleValueTranslatorTest
///////////////////////////////////////////////////////////
possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
MemoryRecordStore.setCollectStatistics(true);
MemoryRecordStore.resetStatistics();
possibleValueTranslator.translatePossibleValue(shapeField, 1);
possibleValueTranslator.translatePossibleValue(shapeField, 2);
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 queries so far");

316
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Shape.java Normal file
View File

@ -0,0 +1,316 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.data.testentities;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
/*******************************************************************************
**
*******************************************************************************/
public class Shape extends QRecordEntity
{
@QField()
private Integer id;
@QField()
private Instant createDate;
@QField()
private Instant modifyDate;
@QField()
private String name;
@QField()
private String type;
@QField()
private Integer noOfSides;
@QField()
private Boolean isPolygon;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public Shape()
{
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public Shape(QRecord record)
{
populateFromQRecord(record);
}
/*******************************************************************************
** Getter for id
**
*******************************************************************************/
public Integer getId()
{
return id;
}
/*******************************************************************************
** Setter for id
**
*******************************************************************************/
public void setId(Integer id)
{
this.id = id;
}
/*******************************************************************************
** Fluent setter for id
**
*******************************************************************************/
public Shape withId(Integer id)
{
this.id = id;
return (this);
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public Shape withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for createDate
**
*******************************************************************************/
public Instant getCreateDate()
{
return createDate;
}
/*******************************************************************************
** Setter for createDate
**
*******************************************************************************/
public void setCreateDate(Instant createDate)
{
this.createDate = createDate;
}
/*******************************************************************************
** Fluent setter for createDate
**
*******************************************************************************/
public Shape withCreateDate(Instant createDate)
{
this.createDate = createDate;
return (this);
}
/*******************************************************************************
** Getter for modifyDate
**
*******************************************************************************/
public Instant getModifyDate()
{
return modifyDate;
}
/*******************************************************************************
** Setter for modifyDate
**
*******************************************************************************/
public void setModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
}
/*******************************************************************************
** Fluent setter for modifyDate
**
*******************************************************************************/
public Shape withModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
return (this);
}
/*******************************************************************************
** Getter for type
**
*******************************************************************************/
public String getType()
{
return type;
}
/*******************************************************************************
** Setter for type
**
*******************************************************************************/
public void setType(String type)
{
this.type = type;
}
/*******************************************************************************
** Fluent setter for type
**
*******************************************************************************/
public Shape withType(String type)
{
this.type = type;
return (this);
}
/*******************************************************************************
** Getter for noOfSides
**
*******************************************************************************/
public Integer getNoOfSides()
{
return noOfSides;
}
/*******************************************************************************
** Setter for noOfSides
**
*******************************************************************************/
public void setNoOfSides(Integer noOfSides)
{
this.noOfSides = noOfSides;
}
/*******************************************************************************
** Fluent setter for noOfSides
**
*******************************************************************************/
public Shape withNoOfSides(Integer noOfSides)
{
this.noOfSides = noOfSides;
return (this);
}
/*******************************************************************************
** Getter for isPolygon
**
*******************************************************************************/
public Boolean getIsPolygon()
{
return isPolygon;
}
/*******************************************************************************
** Setter for isPolygon
**
*******************************************************************************/
public void setIsPolygon(Boolean isPolygon)
{
this.isPolygon = isPolygon;
}
/*******************************************************************************
** Fluent setter for isPolygon
**
*******************************************************************************/
public Shape withIsPolygon(Boolean isPolygon)
{
this.isPolygon = isPolygon;
return (this);
}
}

16
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java
View File

@ -55,7 +55,7 @@ class TableSyncProcessTest
**
*******************************************************************************/
@Test
void test() throws QException
void test() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
@ -86,10 +86,6 @@ class TableSyncProcessTest
qInstance.addProcess(TableSyncProcess.processMetaDataBuilder(false)
.withName(PROCESS_NAME)
.withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withDestinationTable(TABLE_NAME_PEOPLE_SYNC)
.withSourceTableKeyField("id")
.withDestinationTableForeignKeyField("sourcePersonId")
.withSyncTransformStepClass(PersonTransformClass.class)
.getProcessMetaData());
@ -142,6 +138,16 @@ class TableSyncProcessTest
return (destinationRecord);
}
@Override
protected SyncProcessConfig getSyncProcessConfig()
{
SyncProcessConfig syncProcessConfig = new SyncProcessConfig(TestUtils.TABLE_NAME_PERSON_MEMORY, "id", "peopleSync", "sourcePersonId");
syncProcessConfig.noop();
return (syncProcessConfig);
}
}
}

128
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtilsTest.java
View File

@ -30,12 +30,14 @@ import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.testentities.Shape;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
@ -46,7 +48,9 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -329,6 +333,11 @@ class GeneralProcessUtilsTest
assertEquals(3, recordMapByFirstName.size());
assertEquals(1, recordMapByFirstName.get("Darin").getValueInteger("id"));
assertEquals(3, recordMapByFirstName.get("Tim").getValueInteger("id"));
Map<String, QRecord> recordMapByFirstNameAsString = GeneralProcessUtils.loadTableToMap(queryInput, TestUtils.TABLE_NAME_PERSON_MEMORY, String.class, "firstName");
assertEquals(3, recordMapByFirstName.size());
assertEquals(1, recordMapByFirstName.get("Darin").getValueInteger("id"));
assertEquals(3, recordMapByFirstName.get("Tim").getValueInteger("id"));
}
@ -354,4 +363,123 @@ class GeneralProcessUtilsTest
assertEquals(1, map.get("Darin").size());
assertEquals(2, map.get("James").size());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetRecordByFieldOrElseThrow() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
assertNotNull(GeneralProcessUtils.getRecordByFieldOrElseThrow(new AbstractActionInput(instance, new QSession()), TestUtils.TABLE_NAME_SHAPE, "name", "Triangle"));
assertThrows(QException.class, () -> GeneralProcessUtils.getRecordByFieldOrElseThrow(new AbstractActionInput(instance, new QSession()), TestUtils.TABLE_NAME_SHAPE, "name", "notAShape"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetRecordByPrimaryKey() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
assertTrue(GeneralProcessUtils.getRecordByPrimaryKey(actionInput, TestUtils.TABLE_NAME_SHAPE, 1).isPresent());
assertFalse(GeneralProcessUtils.getRecordByPrimaryKey(actionInput, TestUtils.TABLE_NAME_SHAPE, -1).isPresent());
assertNotNull(GeneralProcessUtils.getRecordByPrimaryKeyOrElseThrow(actionInput, TestUtils.TABLE_NAME_SHAPE, 1));
assertThrows(QException.class, () -> GeneralProcessUtils.getRecordByPrimaryKeyOrElseThrow(actionInput, TestUtils.TABLE_NAME_SHAPE, -1));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCount() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
assertEquals(3, GeneralProcessUtils.count(actionInput, TestUtils.TABLE_NAME_SHAPE, null));
assertEquals(1, GeneralProcessUtils.count(actionInput, TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 2))));
assertEquals(0, GeneralProcessUtils.count(actionInput, TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IS_BLANK))));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetEntityByField() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
assertEquals("Triangle", GeneralProcessUtils.getEntityByField(actionInput, TestUtils.TABLE_NAME_SHAPE, "name", "Triangle", Shape.class).get().getName());
assertFalse(GeneralProcessUtils.getEntityByField(actionInput, TestUtils.TABLE_NAME_SHAPE, "name", "notAShape", Shape.class).isPresent());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testLoadTableAsEntities() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
List<Shape> shapes = GeneralProcessUtils.loadTable(actionInput, TestUtils.TABLE_NAME_SHAPE, Shape.class);
assertEquals(3, shapes.size());
assertTrue(shapes.get(0) instanceof Shape);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testLoadTableToMapAsEntities() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
Map<Serializable, Shape> map = GeneralProcessUtils.loadTableToMap(actionInput, TestUtils.TABLE_NAME_SHAPE, "id", Shape.class);
assertEquals(3, map.size());
assertTrue(map.get(1) instanceof Shape);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testRecordsToEntities() throws QException
{
List<Shape> shapes = GeneralProcessUtils.recordsToEntities(Shape.class, List.of(
new QRecord().withValue("id", 99).withValue("name", "round"),
new QRecord().withValue("id", 98).withValue("name", "flat")
));
assertEquals(2, shapes.size());
assertEquals(99, shapes.get(0).getId());
assertEquals("round", shapes.get(0).getName());
}
}

123
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java Normal file
View File

@ -0,0 +1,123 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.utils;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for RecordLookupHelper
*******************************************************************************/
class RecordLookupHelperTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
MemoryRecordStore.resetStatistics();
MemoryRecordStore.setCollectStatistics(true);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithoutPreload() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(qInstance);
RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput(qInstance, new QSession()));
MemoryRecordStore.setCollectStatistics(true);
assertEquals(2, recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "Square"));
assertEquals(2, recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "Square", Integer.class));
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
assertEquals("Circle", recordLookupHelper.getRecordValue(TestUtils.TABLE_NAME_SHAPE, "name", "id", 3));
assertEquals("Circle", recordLookupHelper.getRecordValue(TestUtils.TABLE_NAME_SHAPE, "name", "id", 3, String.class));
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
assertNull(recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "notAShape"));
assertNull(recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "notAShape", Integer.class));
assertEquals(3, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithPreload() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(qInstance);
RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput(qInstance, new QSession()));
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name");
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
assertNotNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Triangle"));
assertEquals(1, recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Triangle").getValueInteger("id"));
assertEquals("Triangle", recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Triangle").getValueString("name"));
assertEquals(2, recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Square").getValueInteger("id"));
assertEquals("Square", recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Square").getValueString("name"));
assertEquals(3, recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Circle").getValueInteger("id"));
assertEquals("Circle", recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Circle").getValueString("name"));
/////////////////////////////////////////////////////
// all those gets should run no additional queries //
/////////////////////////////////////////////////////
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
////////////////////////////////////////////////////////////////////
// make sure we don't re-do the query in a second call to preload //
////////////////////////////////////////////////////////////////////
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name");
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
///////////////////////////////////////////////////
// make sure we can preload by a different field //
///////////////////////////////////////////////////
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "id");
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
}
}

14
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
View File

@ -193,9 +193,17 @@ public abstract class AbstractRDBMSAction implements QActionInterface
////////////////////////////////////////////////////////////
// find the join in the instance, to see the 'on' clause //
////////////////////////////////////////////////////////////
List<String> joinClauseList = new ArrayList<>();
String leftTableName = joinsContext.resolveTableNameOrAliasToTableName(queryJoin.getLeftTableOrAlias());
QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> findJoinMetaData(instance, leftTableName, queryJoin.getRightTable()));
List<String> joinClauseList = new ArrayList<>();
String leftTableName = joinsContext.resolveTableNameOrAliasToTableName(queryJoin.getLeftTableOrAlias());
QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () ->
{
QJoinMetaData found = findJoinMetaData(instance, leftTableName, queryJoin.getRightTable());
if(found == null)
{
throw (new RuntimeException("Could not find a join between tables [" + leftTableName + "][" + queryJoin.getRightTable() + "]"));
}
return (found);
});
for(JoinOn joinOn : joinMetaData.getJoinOns())
{
QTableMetaData leftTable = instance.getTable(joinMetaData.getLeftTable());

220
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -184,6 +184,7 @@ public class QJavalinImplementation
service = Javalin.create().start(port);
service.routes(getRoutes());
service.before(QJavalinImplementation::hotSwapQInstance);
service.before((Context context) -> context.header("Content-Type", "application/json"));
}
@ -815,26 +816,9 @@ public class QJavalinImplementation
String filter = context.queryParam("filter");
Integer limit = integerQueryParam(context, "limit");
/////////////////////////////////////////////////////////////////////////////////////////
// if a format query param wasn't given, then try to get file extension from file name //
/////////////////////////////////////////////////////////////////////////////////////////
if(!StringUtils.hasContent(format) && optionalFilename.isPresent() && StringUtils.hasContent(optionalFilename.get()))
ReportFormat reportFormat = getReportFormat(context, optionalFilename, format);
if(reportFormat == null)
{
String filename = optionalFilename.get();
if(filename.contains("."))
{
format = filename.substring(filename.lastIndexOf(".") + 1);
}
}
ReportFormat reportFormat;
try
{
reportFormat = ReportFormat.fromString(format);
}
catch(QUserFacingException e)
{
handleException(HttpStatus.Code.BAD_REQUEST, context, e);
return;
}
@ -861,55 +845,21 @@ public class QJavalinImplementation
exportInput.setQueryFilter(JsonUtils.toObject(filter, QQueryFilter.class));
}
///////////////////////////////////////////////////////////////////////////////////////////////////////
// set up the I/O pipe streams. //
// Critically, we must NOT open the outputStream in a try-with-resources. The thread that writes to //
// the stream must close it when it's done writing. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
PipedOutputStream pipedOutputStream = new PipedOutputStream();
PipedInputStream pipedInputStream = new PipedInputStream();
pipedOutputStream.connect(pipedInputStream);
exportInput.setReportOutputStream(pipedOutputStream);
ExportAction exportAction = new ExportAction();
exportAction.preExecute(exportInput);
/////////////////////////////////////////////////////////////////////////////////////////////////////
// start the async job. //
// Critically, this must happen before the pipedInputStream is passed to the javalin result method //
/////////////////////////////////////////////////////////////////////////////////////////////////////
new AsyncJobManager().startJob("Javalin>ReportAction", (o) ->
UnsafeFunction<PipedOutputStream, ExportAction> preAction = (PipedOutputStream pos) ->
{
try
{
exportAction.execute(exportInput);
return (true);
}
catch(Exception e)
{
pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes());
pipedOutputStream.close();
return (false);
}
});
exportInput.setReportOutputStream(pos);
////////////////////////////////////////////
// set the response content type & stream //
////////////////////////////////////////////
context.contentType(reportFormat.getMimeType());
context.header("Content-Disposition", "filename=" + filename);
context.result(pipedInputStream);
ExportAction exportAction = new ExportAction();
exportAction.preExecute(exportInput);
return (exportAction);
};
////////////////////////////////////////////////////////////////////////////////////////////
// we'd like to check to see if the job failed, and if so, to give the user an error... //
// but if we "block" here, then piped streams seem to never flush, so we deadlock things. //
////////////////////////////////////////////////////////////////////////////////////////////
// AsyncJobStatus asyncJobStatus = asyncJobManager.waitForJob(jobUUID);
// if(asyncJobStatus.getState().equals(AsyncJobState.ERROR))
// {
// System.out.println("Well, here we are...");
// throw (new QUserFacingException("Error running report: " + asyncJobStatus.getCaughtException().getMessage()));
// }
UnsafeConsumer<ExportAction> execute = (ExportAction exportAction) ->
{
exportAction.execute(exportInput);
};
runStreamedExportOrReport(context, reportFormat, filename, preAction, execute);
}
catch(Exception e)
{
@ -919,6 +869,122 @@ public class QJavalinImplementation
/*******************************************************************************
**
*******************************************************************************/
@FunctionalInterface
public interface UnsafeFunction<T, R>
{
/*******************************************************************************
**
*******************************************************************************/
R run(T t) throws Exception;
}
/*******************************************************************************
**
*******************************************************************************/
@FunctionalInterface
public interface UnsafeConsumer<T>
{
/*******************************************************************************
**
*******************************************************************************/
void run(T t) throws Exception;
}
/*******************************************************************************
**
*******************************************************************************/
public static <T> void runStreamedExportOrReport(Context context, ReportFormat reportFormat, String filename, UnsafeFunction<PipedOutputStream, T> preAction, UnsafeConsumer<T> executor) throws Exception
{
///////////////////////////////////////////////////////////////////////////////////////////////////////
// set up the I/O pipe streams. //
// Critically, we must NOT open the outputStream in a try-with-resources. The thread that writes to //
// the stream must close it when it's done writing. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
PipedOutputStream pipedOutputStream = new PipedOutputStream();
PipedInputStream pipedInputStream = new PipedInputStream();
pipedOutputStream.connect(pipedInputStream);
T t = preAction.run(pipedOutputStream);
/////////////////////////////////////////////////////////////////////////////////////////////////////
// start the async job. //
// Critically, this must happen before the pipedInputStream is passed to the javalin result method //
/////////////////////////////////////////////////////////////////////////////////////////////////////
new AsyncJobManager().startJob("Javalin>ExportAction", (o) ->
{
try
{
executor.run(t);
return (true);
}
catch(Exception e)
{
pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes());
pipedOutputStream.close();
return (false);
}
});
////////////////////////////////////////////
// set the response content type & stream //
////////////////////////////////////////////
context.contentType(reportFormat.getMimeType());
context.header("Content-Disposition", "filename=" + filename);
context.result(pipedInputStream);
////////////////////////////////////////////////////////////////////////////////////////////
// we'd like to check to see if the job failed, and if so, to give the user an error... //
// but if we "block" here, then piped streams seem to never flush, so we deadlock things. //
////////////////////////////////////////////////////////////////////////////////////////////
// AsyncJobStatus asyncJobStatus = asyncJobManager.waitForJob(jobUUID);
// if(asyncJobStatus.getState().equals(AsyncJobState.ERROR))
// {
// System.out.println("Well, here we are...");
// throw (new QUserFacingException("Error running report: " + asyncJobStatus.getCaughtException().getMessage()));
// }
}
/*******************************************************************************
**
*******************************************************************************/
public static ReportFormat getReportFormat(Context context, Optional<String> optionalFilename, String format)
{
/////////////////////////////////////////////////////////////////////////////////////////
// if a format query param wasn't given, then try to get file extension from file name //
/////////////////////////////////////////////////////////////////////////////////////////
if(!StringUtils.hasContent(format) && optionalFilename.isPresent() && StringUtils.hasContent(optionalFilename.get()))
{
String filename = optionalFilename.get();
if(filename.contains("."))
{
format = filename.substring(filename.lastIndexOf(".") + 1);
}
}
ReportFormat reportFormat;
try
{
reportFormat = ReportFormat.fromString(format);
}
catch(QUserFacingException e)
{
handleException(HttpStatus.Code.BAD_REQUEST, context, e);
return null;
}
return reportFormat;
}
/*******************************************************************************
**
*******************************************************************************/
@ -997,21 +1063,21 @@ public class QJavalinImplementation
{
if(userFacingException instanceof QNotFoundException)
{
int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.NOT_FOUND).getCode();
context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
statusCode = Objects.requireNonNullElse(statusCode, HttpStatus.Code.NOT_FOUND); // 404
respondWithError(context, statusCode, userFacingException.getMessage());
}
else
{
LOG.info("User-facing exception", e);
int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode();
context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
statusCode = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR); // 500
respondWithError(context, statusCode, userFacingException.getMessage());
}
}
else
{
if(e instanceof QAuthenticationException)
{
context.status(HttpStatus.UNAUTHORIZED_401).result("{\"error\":\"" + e.getMessage() + "\"}");
respondWithError(context, HttpStatus.Code.UNAUTHORIZED, e.getMessage()); // 401
return;
}
@ -1019,13 +1085,23 @@ public class QJavalinImplementation
// default exception handling //
////////////////////////////////
LOG.warn("Exception in javalin request", e);
int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode();
context.status(code).result("{\"error\":\"" + e.getClass().getSimpleName() + " (" + e.getMessage() + ")\"}");
respondWithError(context, HttpStatus.Code.INTERNAL_SERVER_ERROR, e.getClass().getSimpleName() + " (" + e.getMessage() + ")"); // 500
}
}
/*******************************************************************************
**
*******************************************************************************/
public static void respondWithError(Context context, HttpStatus.Code statusCode, String errorMessage)
{
context.status(statusCode.getCode());
context.result(JsonUtils.toJson(Map.of("error", errorMessage)));
}
/*******************************************************************************
** Returns Integer if context has a valid int query parameter by the given name,
** Returns null if no param (or empty value).

129
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java
View File

@ -27,6 +27,7 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.PipedOutputStream;
import java.io.Serializable;
import java.time.LocalDate;
import java.util.ArrayList;
@ -34,6 +35,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@ -45,15 +47,19 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
import com.kingsrook.qqq.backend.core.actions.async.JobGoingAsyncException;
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@ -62,6 +68,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
@ -70,12 +77,14 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import io.javalin.apibuilder.EndpointGroup;
import io.javalin.http.Context;
import io.javalin.http.UploadedFile;
import org.apache.commons.lang.NotImplementedException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.jetty.http.HttpStatus;
import static io.javalin.apibuilder.ApiBuilder.get;
import static io.javalin.apibuilder.ApiBuilder.path;
import static io.javalin.apibuilder.ApiBuilder.post;
@ -115,11 +124,131 @@ public class QJavalinProcessHandler
});
});
get("/download/{file}", QJavalinProcessHandler::downloadFile);
path("/reports", () ->
{
path("/{reportName}", () ->
{
get("", QJavalinProcessHandler::reportWithoutFilename);
get("/{fileName}", QJavalinProcessHandler::reportWithFilename);
});
});
});
}
/*******************************************************************************
**
*******************************************************************************/
private static void reportWithFilename(Context context)
{
String filename = context.pathParam("fileName");
report(context, Optional.of(filename));
}
/*******************************************************************************
**
*******************************************************************************/
private static void reportWithoutFilename(Context context)
{
report(context, Optional.empty());
}
/*******************************************************************************
**
*******************************************************************************/
private static void report(Context context, Optional<String> optionalFilename)
{
try
{
//////////////////////////////////////////
// read params from the request context //
//////////////////////////////////////////
String reportName = context.pathParam("reportName");
String format = context.queryParam("format");
ReportFormat reportFormat = QJavalinImplementation.getReportFormat(context, optionalFilename, format);
if(reportFormat == null)
{
return;
}
String filename = optionalFilename.orElse(reportName + "." + reportFormat.toString().toLowerCase(Locale.ROOT));
/////////////////////////////////////////////
// set up the report action's input object //
/////////////////////////////////////////////
ReportInput reportInput = new ReportInput(QJavalinImplementation.qInstance);
QJavalinImplementation.setupSession(context, reportInput);
reportInput.setReportFormat(reportFormat);
reportInput.setReportName(reportName);
reportInput.setInputValues(null); // todo!
reportInput.setFilename(filename);
QReportMetaData report = QJavalinImplementation.qInstance.getReport(reportName);
if(report == null)
{
throw (new QNotFoundException("Report [" + reportName + "] is not found."));
}
//////////////////////////////////////////////////////////////
// process the report's input fields, from the query string //
//////////////////////////////////////////////////////////////
for(QFieldMetaData inputField : CollectionUtils.nonNullList(report.getInputFields()))
{
try
{
boolean setValue = false;
if(context.queryParamMap().containsKey(inputField.getName()))
{
String value = context.queryParamMap().get(inputField.getName()).get(0);
Serializable typedValue = ValueUtils.getValueAsFieldType(inputField.getType(), value);
reportInput.addInputValue(inputField.getName(), typedValue);
setValue = true;
}
if(inputField.getIsRequired() && !setValue)
{
QJavalinImplementation.respondWithError(context, HttpStatus.Code.BAD_REQUEST, "Missing query param value for required input field: [" + inputField.getName() + "]");
return;
}
}
catch(Exception e)
{
QJavalinImplementation.respondWithError(context, HttpStatus.Code.BAD_REQUEST, "Error processing query param [" + inputField.getName() + "]: " + e.getClass().getSimpleName() + " (" + e.getMessage() + ")");
return;
}
}
QJavalinImplementation.UnsafeFunction<PipedOutputStream, GenerateReportAction> preAction = (PipedOutputStream pos) ->
{
reportInput.setReportOutputStream(pos);
GenerateReportAction reportAction = new GenerateReportAction();
// any pre-action?? export uses this for "too many rows" checks...
return (reportAction);
};
QJavalinImplementation.UnsafeConsumer<GenerateReportAction> execute = (GenerateReportAction generateReportAction) ->
{
generateReportAction.execute(reportInput);
};
QJavalinImplementation.runStreamedExportOrReport(context, reportFormat, filename, preAction, execute);
}
catch(Exception e)
{
QJavalinImplementation.handleException(context, e);
}
}
/*******************************************************************************
**
*******************************************************************************/

4
qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
View File

@ -471,7 +471,7 @@ class QJavalinImplementationTest extends QJavalinTestBase
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/MyPersonExport.csv").asString();
assertEquals(200, response.getStatus());
assertEquals("text/csv", response.getHeaders().get("Content-Type").get(0));
assertEquals("text/csv;charset=utf-8", response.getHeaders().get("Content-Type").get(0));
assertEquals("filename=MyPersonExport.csv", response.getHeaders().get("Content-Disposition").get(0));
String[] csvLines = response.getBody().split("\n");
assertEquals(6, csvLines.length);
@ -500,7 +500,7 @@ class QJavalinImplementationTest extends QJavalinTestBase
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/?format=xlsx").asString();
assertEquals(200, response.getStatus());
assertEquals(ReportFormat.XLSX.getMimeType(), response.getHeaders().get("Content-Type").get(0));
assertEquals(ReportFormat.XLSX.getMimeType() + ";charset=utf-8", response.getHeaders().get("Content-Type").get(0));
assertEquals("filename=person.xlsx", response.getHeaders().get("Content-Disposition").get(0));
}

56
qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java
View File

@ -35,6 +35,7 @@ import kong.unirest.Unirest;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@ -476,4 +477,59 @@ class QJavalinProcessHandlerTest extends QJavalinTestBase
getProcessRecords(processUUID, 0, 5, 5);
}
/*******************************************************************************
** test running a report
**
*******************************************************************************/
@Test
public void test_report()
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/reports/personsReport?format=csv&firstNamePrefix=D").asString();
assertEquals(200, response.getStatus());
assertThat(response.getHeaders().get("Content-Type").get(0)).contains("text/csv");
assertThat(response.getHeaders().get("Content-Disposition").get(0)).contains("filename=personsReport.csv");
String csv = response.getBody();
System.out.println(csv);
assertThat(csv).contains("""
"Id","First Name","Last Name\"""");
assertThat(csv).contains("""
"1","Darin","Kelkhoff\"""");
}
/*******************************************************************************
** test running a report
**
*******************************************************************************/
@Test
public void test_reportMissingFormat()
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/reports/personsReport?firstNamePrefix=D").asString();
assertEquals(400, response.getStatus());
assertThat(response.getHeaders().get("Content-Type").get(0)).contains("application/json");
JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
assertThat(jsonObject.getString("error")).contains("Report format was not specified");
}
/*******************************************************************************
** test running a report by filename
**
*******************************************************************************/
@Test
public void test_reportWithFileName()
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/reports/personsReport/myFile.json?firstNamePrefix=D").asString();
assertEquals(200, response.getStatus());
assertThat(response.getHeaders().get("Content-Type").get(0)).contains("application/json");
assertThat(response.getHeaders().get("Content-Disposition").get(0)).contains("filename=myFile.json");
// JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
// System.out.println(jsonObject);
JSONArray jsonArray = JsonUtils.toJSONArray(response.getBody());
}
}

55
qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java
View File

@ -30,6 +30,9 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -48,6 +51,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
@ -66,6 +74,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
*******************************************************************************/
public class TestUtils
{
public static final String TABLE_NAME_PERSON = "person";
public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive";
public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep";
public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow";
@ -129,6 +139,7 @@ public class TestUtils
qInstance.addProcess(defineProcessSimpleSleep());
qInstance.addProcess(defineProcessScreenThenSleep());
qInstance.addProcess(defineProcessSimpleThrow());
qInstance.addReport(definePersonsReport());
qInstance.addPossibleValueSource(definePossibleValueSourcePerson());
defineWidgets(qInstance);
@ -210,7 +221,7 @@ public class TestUtils
public static QTableMetaData defineTablePerson()
{
return new QTableMetaData()
.withName("person")
.withName(TABLE_NAME_PERSON)
.withLabel("Person")
.withRecordLabelFormat("%s %s")
.withRecordLabelFields("firstName", "lastName")
@ -222,7 +233,7 @@ public class TestUtils
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("partnerPersonId", QFieldType.INTEGER).withBackendName("partner_person_id").withPossibleValueSourceName("person"))
.withField(new QFieldMetaData("partnerPersonId", QFieldType.INTEGER).withBackendName("partner_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withField(new QFieldMetaData("testScriptId", QFieldType.INTEGER).withBackendName("test_script_id"))
.withAssociatedScript(new AssociatedScript()
@ -240,7 +251,7 @@ public class TestUtils
{
return new QProcessMetaData()
.withName("greet")
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addStep(new QBackendStepMetaData()
.withName("prepare")
.withCode(new QCodeReference()
@ -248,14 +259,14 @@ public class TestUtils
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
.withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON))
.withFieldList(List.of(
new QFieldMetaData("greetingPrefix", QFieldType.STRING),
new QFieldMetaData("greetingSuffix", QFieldType.STRING)
)))
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
@ -271,7 +282,7 @@ public class TestUtils
{
return new QProcessMetaData()
.withName(PROCESS_NAME_GREET_PEOPLE_INTERACTIVE)
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addStep(new QFrontendStepMetaData()
.withName("setup")
@ -286,14 +297,14 @@ public class TestUtils
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
.withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON))
.withFieldList(List.of(
new QFieldMetaData("greetingPrefix", QFieldType.STRING),
new QFieldMetaData("greetingSuffix", QFieldType.STRING)
)))
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
@ -313,9 +324,9 @@ public class TestUtils
private static QPossibleValueSource definePossibleValueSourcePerson()
{
return (new QPossibleValueSource()
.withName("person")
.withName(TABLE_NAME_PERSON)
.withType(QPossibleValueSourceType.TABLE)
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.withValueFormatAndFields(PVSValueFormatAndFields.LABEL_PARENS_ID)
.withOrderByField("id")
);
@ -469,4 +480,28 @@ public class TestUtils
}
}
/*******************************************************************************
**
*******************************************************************************/
private static QReportMetaData definePersonsReport()
{
return new QReportMetaData()
.withName("personsReport")
.withInputField(new QFieldMetaData("firstNamePrefix", QFieldType.STRING))
.withDataSource(new QReportDataSource()
.withSourceTable(TABLE_NAME_PERSON)
.withQueryFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.STARTS_WITH, "${input.firstNamePrefix}")))
)
.withView(new QReportView()
.withType(ReportType.TABLE)
.withColumns(List.of(
new QReportField("id"),
new QReportField("firstName"),
new QReportField("lastName")
))
);
}
}