QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updated interface in sync processes; more status updates in ETL processes; Basepull only update timestamp if ran as basepull; javalin report endpoint;

Browse Source
This commit is contained in:
darin.kelkhoff
2022-12-19 10:00:29 -06:00
parent 1b672afcd0
commit e1c53b9d48
34 changed files with 1655 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java
View File

@ -80,6 +80,7 @@ public class RunProcessAction
// indicator that the timestamp field should be updated - e.g., the execute step is finished. //
////////////////////////////////////////////////////////////////////////////////////////////////
public static final String BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD = "basepullReadyToUpdateTimestamp";
public static final String BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD = "basepullDidQueryUsingTimestamp";
@ -190,11 +191,14 @@ public class RunProcessAction
}
}
////////////////////////////////////////////////////////////////////////////////////
// if 'basepull' style process, update the stored basepull timestamp //
// but only when we've been signaled to do so - i.e., after an Execute step runs. //
////////////////////////////////////////////////////////////////////////////////////
if(basepullConfiguration != null && BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD))))
///////////////////////////////////////////////////////////////////////////
// if 'basepull' style process, update the stored basepull timestamp //
// but only when we've been signaled to do so - i.e., only if we did our //
// query using the timestamp field, and only after an Execute step runs. //
///////////////////////////////////////////////////////////////////////////
if(basepullConfiguration != null
&& BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD)))
&& BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD))))
{
storeLastRunTime(runProcessInput, process, basepullConfiguration);
}

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java
View File

@ -179,6 +179,17 @@ public class GenerateReportAction
}
outputSummaries(reportInput);
reportStreamer.finish();
try
{
reportInput.getReportOutputStream().close();
}
catch(Exception e)
{
throw (new QReportingException("Error completing report", e));
}
}
@ -527,8 +538,6 @@ public class GenerateReportAction
reportStreamer.addTotalsRow(summaryOutput.totalRow);
}
}
reportStreamer.finish();
}

155
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java Normal file
View File

@ -0,0 +1,155 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** JSON export format implementation
*******************************************************************************/
public class JsonExportStreamer implements ExportStreamerInterface
{
private static final Logger LOG = LogManager.getLogger(JsonExportStreamer.class);
private ExportInput exportInput;
private QTableMetaData table;
private List<QFieldMetaData> fields;
private OutputStream outputStream;
private boolean needComma = false;
/*******************************************************************************
**
*******************************************************************************/
public JsonExportStreamer()
{
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
{
this.exportInput = exportInput;
this.fields = fields;
table = exportInput.getTable();
outputStream = this.exportInput.getReportOutputStream();
try
{
outputStream.write("[".getBytes(StandardCharsets.UTF_8));
}
catch(IOException e)
{
throw (new QReportingException("Error starting report output", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void addRecords(List<QRecord> qRecords) throws QReportingException
{
LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
for(QRecord qRecord : qRecords)
{
writeRecord(qRecord);
}
}
/*******************************************************************************
**
*******************************************************************************/
private void writeRecord(QRecord qRecord) throws QReportingException
{
try
{
if(needComma)
{
outputStream.write(",".getBytes(StandardCharsets.UTF_8));
}
String json = JsonUtils.toJson(qRecord);
outputStream.write(json.getBytes(StandardCharsets.UTF_8));
outputStream.flush(); // todo - less often?
needComma = true;
}
catch(Exception e)
{
throw (new QReportingException("Error writing JSON report", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void addTotalsRow(QRecord record) throws QReportingException
{
writeRecord(record);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void finish() throws QReportingException
{
try
{
outputStream.write("]".getBytes(StandardCharsets.UTF_8));
}
catch(IOException e)
{
throw (new QReportingException("Error ending report output", e));
}
}
}

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java
View File

@ -38,7 +38,7 @@ import org.apache.logging.log4j.Logger;
** Base input class for all Q actions.
**
*******************************************************************************/
public abstract class AbstractActionInput
public class AbstractActionInput
{
private static final Logger LOG = LogManager.getLogger(AbstractActionInput.class);
@ -69,6 +69,17 @@ public abstract class AbstractActionInput
/*******************************************************************************
**
*******************************************************************************/
public AbstractActionInput(QInstance instance, QSession session)
{
this(instance);
this.session = session;
}
/*******************************************************************************
** performance instance validation (if not previously done).
*******************************************************************************/

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java
View File

@ -27,6 +27,7 @@ import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.actions.reporting.CsvExportStreamer;
import com.kingsrook.qqq.backend.core.actions.reporting.ExcelExportStreamer;
import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
import com.kingsrook.qqq.backend.core.actions.reporting.JsonExportStreamer;
import com.kingsrook.qqq.backend.core.actions.reporting.ListOfMapsExportStreamer;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -39,6 +40,7 @@ import org.dhatim.fastexcel.Worksheet;
public enum ReportFormat
{
XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelExportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
JSON(null, null, JsonExportStreamer::new, "application/json"),
CSV(null, null, CsvExportStreamer::new, "text/csv"),
LIST_OF_MAPS(null, null, ListOfMapsExportStreamer::new, null);

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.reporting;
import java.io.OutputStream;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -118,6 +119,20 @@ public class ReportInput extends AbstractTableActionInput
/*******************************************************************************
**
*******************************************************************************/
public void addInputValue(String key, Serializable value)
{
if(this.inputValues == null)
{
this.inputValues = new HashMap<>();
}
this.inputValues.put(key, value);
}
/*******************************************************************************
** Getter for filename
**

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java
View File

@ -104,38 +104,7 @@ public class QFrontendTableMetaData
Set<Capability> enabledCapabilities = new HashSet<>();
for(Capability capability : Capability.values())
{
///////////////////////////////////////////////
// by default, every table can do everything //
///////////////////////////////////////////////
boolean hasCapability = true;
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the table's backend says the capability is disabled, then by default, then the capability is disabled... //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(backend.getDisabledCapabilities().contains(capability))
{
hasCapability = false;
/////////////////////////////////////////////////////////////////
// unless the table overrides that and says that it IS enabled //
/////////////////////////////////////////////////////////////////
if(table.getEnabledCapabilities().contains(capability))
{
hasCapability = true;
}
}
else
{
/////////////////////////////////////////////////////////////////////////////////////////
// if the backend doesn't specify the capability, then disable it if the table says so //
/////////////////////////////////////////////////////////////////////////////////////////
if(table.getDisabledCapabilities().contains(capability))
{
hasCapability = false;
}
}
if(hasCapability)
if(table.isCapabilityEnabled(backend, capability))
{
///////////////////////////////////////
// todo - check if user is allowed!! //

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
/*******************************************************************************
@ -65,4 +66,16 @@ public class AbstractProcessMetaDataBuilder
.filter(f -> f.getName().equals(fieldName)).findFirst()
.ifPresent(f -> f.setDefaultValue(value));
}
/*******************************************************************************
**
*******************************************************************************/
public AbstractProcessMetaDataBuilder withBasepullConfiguration(BasepullConfiguration basepullConfiguration)
{
processMetaData.setBasepullConfiguration(basepullConfiguration);
return (this);
}
}

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java
View File

@ -34,8 +34,8 @@ public enum Capability
TABLE_INSERT,
TABLE_UPDATE,
TABLE_DELETE
//////////////////////////////////////////////////////////////////////////
// keep these values in sync with AdornmentType.ts in qqq-frontend-core //
//////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////
// keep these values in sync with Capability.ts in qqq-frontend-core //
///////////////////////////////////////////////////////////////////////
}

35
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java
View File

@ -41,6 +41,7 @@ public class QFieldSection
private QIcon icon;
private boolean isHidden = false;
private Integer gridColumns;
@ -328,4 +329,38 @@ public class QFieldSection
return (this);
}
/*******************************************************************************
** Getter for gridColumns
**
*******************************************************************************/
public Integer getGridColumns()
{
return gridColumns;
}
/*******************************************************************************
** Setter for gridColumns
**
*******************************************************************************/
public void setGridColumns(Integer gridColumns)
{
this.gridColumns = gridColumns;
}
/*******************************************************************************
** Fluent setter for gridColumns
**
*******************************************************************************/
public QFieldSection withGridColumns(Integer gridColumns)
{
this.gridColumns = gridColumns;
return (this);
}
}

41
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java
View File

@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizer;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntityField;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
@ -1037,4 +1038,44 @@ public class QTableMetaData implements QAppChildMetaData, Serializable
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public boolean isCapabilityEnabled(QBackendMetaData backend, Capability capability)
{
///////////////////////////////////////////////
// by default, every table can do everything //
///////////////////////////////////////////////
boolean hasCapability = true;
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the table's backend says the capability is disabled, then by default, then the capability is disabled... //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(backend.getDisabledCapabilities().contains(capability))
{
hasCapability = false;
/////////////////////////////////////////////////////////////////
// unless the table overrides that and says that it IS enabled //
/////////////////////////////////////////////////////////////////
if(getEnabledCapabilities().contains(capability))
{
hasCapability = true;
}
}
else
{
/////////////////////////////////////////////////////////////////////////////////////////
// if the backend doesn't specify the capability, then disable it if the table says so //
/////////////////////////////////////////////////////////////////////////////////////////
if(getDisabledCapabilities().contains(capability))
{
hasCapability = false;
}
}
return (hasCapability);
}
}

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java
View File

@ -51,6 +51,7 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
try
{
queryFilter = super.getQueryFilter(runBackendStepInput);
return (queryFilter);
}
catch(QException qe)
{
@ -77,6 +78,12 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
queryFilter.addOrderBy(new QFilterOrderBy(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD)));
/////////////////////////////////////////////////////////////////////////////////////
// put a flag in the process's values, to note that we did use the timestamp field //
// this will later be checked to see if we should update the timestamp too. //
/////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue(RunProcessAction.BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD, true);
return (queryFilter);
}

102
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java
View File

@ -1,102 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for performing the Load step of a basic ETL process using update.
*******************************************************************************/
public class BasicETLLoadAsUpdateFunction implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLLoadAsUpdateFunction.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
//////////////////////////////////////////////////////
// exit early with no-op if no records made it here //
//////////////////////////////////////////////////////
List<QRecord> inputRecords = runBackendStepInput.getRecords();
LOG.info("Received [" + inputRecords.size() + "] records to load using update");
if(CollectionUtils.nullSafeIsEmpty(inputRecords))
{
runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0);
return;
}
/////////////////////////////////////////////////////////////////
// put the destination table name in all records being updated //
/////////////////////////////////////////////////////////////////
String table = runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
for(QRecord record : inputRecords)
{
record.setTableName(table);
}
//////////////////////////////////////////
// run an update request on the records //
//////////////////////////////////////////
int recordsUpdated = 0;
List<QRecord> outputRecords = new ArrayList<>();
int pageSize = 1000; // todo - make this a field?
for(List<QRecord> page : CollectionUtils.getPages(inputRecords, pageSize))
{
LOG.info("Updating a page of [" + page.size() + "] records. Progress: " + recordsUpdated + " loaded out of " + inputRecords.size() + " total");
runBackendStepInput.getAsyncJobCallback().updateStatus("Updating records", recordsUpdated, inputRecords.size());
UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance());
updateInput.setSession(runBackendStepInput.getSession());
updateInput.setTableName(table);
updateInput.setRecords(page);
UpdateAction updateAction = new UpdateAction();
UpdateOutput updateResult = updateAction.execute(updateInput);
outputRecords.addAll(updateResult.getRecords());
recordsUpdated += updateResult.getRecords().size();
}
runBackendStepOutput.setRecords(outputRecords);
runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsUpdated);
}
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
View File

@ -62,6 +62,8 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
try
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Executing Process");
///////////////////////////////////////////////////////
// set up the extract, transform, and load functions //
///////////////////////////////////////////////////////
@ -137,6 +139,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
//////////////////////////////////////////////////////////////////////////////
// set the flag to state that the basepull timestamp should be updated now. //
// (upstream will check if the process was running as a basepull) //
//////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue(RunProcessAction.BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD, true);

30
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java
View File

@ -32,7 +32,11 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -54,6 +58,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Integer limit = PROCESS_OUTPUT_RECORD_LIST_LIMIT; // todo - use a field instead of hard-coded here?
runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Preview");
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the do-full-validation flag has already been set, then do the validation step instead of this one //
@ -69,7 +74,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
if(runBackendStepInput.getFrontendStepBehavior() != null && runBackendStepInput.getFrontendStepBehavior().equals(RunProcessInput.FrontendStepBehavior.SKIP))
{
LOG.debug("Skipping preview because frontent behavior is [" + RunProcessInput.FrontendStepBehavior.SKIP + "].");
LOG.debug("Skipping preview because frontend behavior is [" + RunProcessInput.FrontendStepBehavior.SKIP + "].");
return;
}
@ -91,8 +96,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
extractStep.setRecordPipe(recordPipe);
extractStep.preRun(runBackendStepInput, runBackendStepOutput);
Integer recordCount = extractStep.doCount(runBackendStepInput);
runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount);
countRecords(runBackendStepInput, runBackendStepOutput, extractStep);
AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
transformStep.preRun(runBackendStepInput, runBackendStepOutput);
@ -126,6 +130,26 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
/*******************************************************************************
**
*******************************************************************************/
private void countRecords(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, AbstractExtractStep extractStep) throws QException
{
String sourceTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(sourceTableName);
if(StringUtils.hasContent(sourceTableName))
{
QBackendMetaData sourceTableBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTableName);
if(sourceTable.isCapabilityEnabled(sourceTableBackend, Capability.TABLE_COUNT))
{
Integer recordCount = extractStep.doCount(runBackendStepInput);
runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount);
}
}
}
/*******************************************************************************
**
*******************************************************************************/

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java
View File

@ -81,6 +81,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back
//////////////////////////////////////////////////////////
// basically repeat the preview step, but with no limit //
//////////////////////////////////////////////////////////
runBackendStepInput.getAsyncJobCallback().updateStatus("Validating Records");
RecordPipe recordPipe = new RecordPipe();
AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
extractStep.setLimit(null);

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java
View File

@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
/*******************************************************************************
@ -389,5 +390,17 @@ public class StreamedETLWithFrontendProcess
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Builder withBasepullConfiguration(BasepullConfiguration basepullConfiguration)
{
processMetaData.setBasepullConfiguration(basepullConfiguration);
return (this);
}
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java
View File

@ -65,6 +65,7 @@ public class MockBackendStep implements BackendStep
runBackendStepOutput.setValues(runBackendStepInput.getValues());
runBackendStepOutput.addValue(FIELD_MOCK_VALUE, MOCK_VALUE);
runBackendStepOutput.addValue("noOfPeopleGreeted", runBackendStepInput.getRecords().size());
runBackendStepOutput.addValue(RunProcessAction.BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD, true);
runBackendStepOutput.addValue(RunProcessAction.BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD, true);
if("there".equalsIgnoreCase(runBackendStepInput.getValueString(FIELD_GREETING_SUFFIX)))

163
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java
View File

@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -45,11 +44,15 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer;
import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
import com.kingsrook.qqq.backend.core.processes.utils.RecordLookupHelper;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
@ -69,44 +72,11 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
.withSingularPastMessage("was not synced, because it is ")
.withPluralPastMessage("were not synced, because they are ");
private RunBackendStepInput runBackendStepInput = null;
protected RunBackendStepInput runBackendStepInput = null;
protected RecordLookupHelper recordLookupHelper = null;
private QPossibleValueTranslator possibleValueTranslator;
private Map<String, Map<Serializable, QRecord>> tableMaps = new HashMap<>();
/*******************************************************************************
**
*******************************************************************************/
protected QRecord getRecord(String tableName, String fieldName, Serializable value) throws QException
{
if(!tableMaps.containsKey(tableName))
{
Map<Serializable, QRecord> recordMap = GeneralProcessUtils.loadTableToMap(runBackendStepInput, tableName, fieldName);
tableMaps.put(tableName, recordMap);
}
return (tableMaps.get(tableName).get(value));
}
/*******************************************************************************
**
*******************************************************************************/
protected Serializable getRecordField(String tableName, String fieldName, Serializable value, String outputField) throws QException
{
QRecord record = getRecord(tableName, fieldName, value);
if(record == null)
{
return (null);
}
return (record.getValue(outputField));
}
/*******************************************************************************
@ -121,12 +91,65 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
/*******************************************************************************
** Map a record from the source table to the destination table. e.g., put
** values into the destinationRecord, from the sourceRecord.
**
** The destinationRecord will already be constructed, and will actually already
** be the record being updated, in the case of an update. It'll be empty (newly
** constructed) for an insert.
*******************************************************************************/
public abstract QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException;
/*******************************************************************************
** Specify a list of tableName/keyColumnName pairs to run through
** the preloadRecords method of the recordLookupHelper.
*******************************************************************************/
protected List<Pair<String, String>> getLookupsToPreLoad()
{
return (null);
}
/*******************************************************************************
** Define the query filter to find existing records. e.g., for determining
** insert vs. update. Subclasses may override this to customize the behavior,
** e.g., in case an additional field is needed in the query.
*******************************************************************************/
protected QQueryFilter getExistingRecordQueryFilter(RunBackendStepInput runBackendStepInput, List<Serializable> sourceKeyList)
{
String destinationTableForeignKeyField = getSyncProcessConfig().destinationTableForeignKey;
return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList));
}
/*******************************************************************************
** Define the config for this process - e.g., what fields & tables are used.
*******************************************************************************/
protected abstract SyncProcessConfig getSyncProcessConfig();
/*******************************************************************************
** Record to store the config for this process - e.g., what fields & tables are used.
*******************************************************************************/
public record SyncProcessConfig(String sourceTable, String sourceTableKeyField, String destinationTable, String destinationTableForeignKey)
{
/*******************************************************************************
** artificial method, here to make jacoco see that this class is indeed
** included in test coverage...
*******************************************************************************/
void noop()
{
System.out.println("noop");
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -139,9 +162,33 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
}
this.runBackendStepInput = runBackendStepInput;
String sourceTableKeyField = runBackendStepInput.getValueString(TableSyncProcess.FIELD_SOURCE_TABLE_KEY_FIELD);
String destinationTableForeignKeyField = runBackendStepInput.getValueString(TableSyncProcess.FIELD_DESTINATION_TABLE_FOREIGN_KEY);
String destinationTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE);
if(this.recordLookupHelper == null)
{
initializeRecordLookupHelper(runBackendStepInput);
}
SyncProcessConfig config = getSyncProcessConfig();
String sourceTableKeyField = config.sourceTableKeyField;
String destinationTableForeignKeyField = config.destinationTableForeignKey;
String destinationTableName = config.destinationTable;
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, destinationTableName);
if(!StringUtils.hasContent(sourceTableKeyField))
{
throw (new IllegalStateException("Missing sourceTableKeyField in config for " + getClass().getSimpleName()));
}
if(!StringUtils.hasContent(destinationTableForeignKeyField))
{
throw (new IllegalStateException("Missing destinationTableForeignKey in config for " + getClass().getSimpleName()));
}
if(!StringUtils.hasContent(destinationTableName))
{
throw (new IllegalStateException("Missing destinationTable in config for " + getClass().getSimpleName()));
}
//////////////////////////////////////
// extract keys from source records //
@ -161,9 +208,8 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance());
queryInput.setSession(runBackendStepInput.getSession());
queryInput.setTableName(destinationTableName);
queryInput.setFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList))
);
QQueryFilter filter = getExistingRecordQueryFilter(runBackendStepInput, sourceKeyList);
queryInput.setFilter(filter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
existingRecordsByForeignKey = CollectionUtils.recordsToMap(queryOutput.getRecords(), destinationTableForeignKeyField);
}
@ -171,10 +217,10 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
/////////////////////////////////////////////////////////////////
// foreach source record, build the record we'll insert/update //
/////////////////////////////////////////////////////////////////
QFieldMetaData destinationForeignKeyField = runBackendStepInput.getInstance().getTable(destinationTableName).getField(destinationTableForeignKeyField);
for(QRecord sourceRecord : runBackendStepInput.getRecords())
{
Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField);
QRecord existingRecord = existingRecordsByForeignKey.get(sourceKeyValue);
if(sourceKeyValue == null || "".equals(sourceKeyValue))
{
@ -194,6 +240,14 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
continue;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// look for the existing record - note - we may need to type-convert here, the sourceKey value //
// from the source table to the destinationKey. e.g., if source table had an integer, and the //
// destination has a string. //
/////////////////////////////////////////////////////////////////////////////////////////////////
Serializable sourceKeyValueInTargetFieldType = ValueUtils.getValueAsFieldType(destinationForeignKeyField.getType(), sourceKeyValue);
QRecord existingRecord = existingRecordsByForeignKey.get(sourceKeyValueInTargetFieldType);
QRecord recordToStore;
if(existingRecord != null)
{
@ -227,4 +281,25 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
}
}
/*******************************************************************************
** If needed, init a record lookup helper for this process.
*******************************************************************************/
protected void initializeRecordLookupHelper(RunBackendStepInput runBackendStepInput) throws QException
{
this.recordLookupHelper = new RecordLookupHelper(runBackendStepInput);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if there's only 1 record, don't bother preloading all records - just do the single lookup by the single needed key. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getRecords().size() > 1)
{
for(Pair<String, String> pair : CollectionUtils.nonNullList(getLookupsToPreLoad()))
{
recordLookupHelper.preloadRecords(pair.getA(), pair.getB());
}
}
}
}

76
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java
View File

@ -24,12 +24,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
import java.util.Collections;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertOrUpdateStep;
@ -84,60 +86,28 @@ public class TableSyncProcess
/*******************************************************************************
** Fluent setter for sourceTableKeyField
**
*******************************************************************************/
public Builder withSourceTableKeyField(String sourceTableKeyField)
{
setInputFieldDefaultValue(FIELD_SOURCE_TABLE_KEY_FIELD, sourceTableKeyField);
return (this);
}
/*******************************************************************************
** Fluent setter for destinationTableForeignKeyField
**
*******************************************************************************/
public Builder withDestinationTableForeignKeyField(String destinationTableForeignKeyField)
{
setInputFieldDefaultValue(FIELD_DESTINATION_TABLE_FOREIGN_KEY, destinationTableForeignKeyField);
return (this);
}
/*******************************************************************************
** Fluent setter for transformStepClass
** Fluent setter for transformStepClass. Note - call this method also makes
** sourceTable and destinationTable be set - by getting them from the
** SyncProcessConfig record defined in the step class.
**
*******************************************************************************/
public Builder withSyncTransformStepClass(Class<? extends AbstractTableSyncTransformStep> transformStepClass)
{
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE, new QCodeReference(transformStepClass));
return (this);
}
AbstractTableSyncTransformStep.SyncProcessConfig config;
try
{
AbstractTableSyncTransformStep transformStep = transformStepClass.getConstructor().newInstance();
config = transformStep.getSyncProcessConfig();
}
catch(Exception e)
{
throw (new QRuntimeException("Error setting up process with transform step class: " + transformStepClass.getName(), e));
}
/*******************************************************************************
** Fluent setter for sourceTable
**
*******************************************************************************/
public Builder withSourceTable(String sourceTable)
{
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, sourceTable);
return (this);
}
/*******************************************************************************
** Fluent setter for destinationTable
**
*******************************************************************************/
public Builder withDestinationTable(String destinationTable)
{
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, destinationTable);
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, config.sourceTable());
setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, config.destinationTable());
return (this);
}
@ -204,5 +174,17 @@ public class TableSyncProcess
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public StreamedETLWithFrontendProcess.Builder withBasepullConfiguration(BasepullConfiguration basepullConfiguration)
{
processMetaData.setBasepullConfiguration(basepullConfiguration);
return (this);
}
}
}

23
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java
View File

@ -203,10 +203,15 @@ public class GeneralProcessUtils
*******************************************************************************/
public static Optional<QRecord> getRecordByField(AbstractActionInput parentActionInput, String tableName, String fieldName, Serializable fieldValue) throws QException
{
if(fieldValue == null)
{
return (Optional.empty());
}
QueryInput queryInput = new QueryInput(parentActionInput.getInstance());
queryInput.setSession(parentActionInput.getSession());
queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, List.of(fieldValue))));
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue)));
queryInput.setLimit(1);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
return (queryOutput.getRecords().stream().findFirst());
@ -321,10 +326,26 @@ public class GeneralProcessUtils
** too many rows... Caveat emptor.
*******************************************************************************/
public static Map<Serializable, QRecord> loadTableToMap(AbstractActionInput parentActionInput, String tableName, String keyFieldName) throws QException
{
return (loadTableToMap(parentActionInput, tableName, keyFieldName, (QQueryFilter) null));
}
/*******************************************************************************
** Load rows from a table matching the specified filter, into a map, keyed by the keyFieldName.
**
** Note - null values from the key field are NOT put in the map.
**
** If multiple values are found for the key, they'll squash each other, and only
** one (random) value will appear.
*******************************************************************************/
public static Map<Serializable, QRecord> loadTableToMap(AbstractActionInput parentActionInput, String tableName, String keyFieldName, QQueryFilter filter) throws QException
{
QueryInput queryInput = new QueryInput(parentActionInput.getInstance());
queryInput.setSession(parentActionInput.getSession());
queryInput.setTableName(tableName);
queryInput.setFilter(filter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
List<QRecord> records = queryOutput.getRecords();

153
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java Normal file
View File

@ -0,0 +1,153 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.utils;
import java.io.Serializable;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** Utility to help processes lookup records. Caches lookups - and potentially
** can pre-load entire tables or subsets of tables.
**
*******************************************************************************/
public class RecordLookupHelper
{
private final AbstractActionInput actionInput;
private Map<String, Map<Serializable, QRecord>> foreignRecordMaps = new HashMap<>();
private Set<String> preloadedKeys = new HashSet<>();
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public RecordLookupHelper(AbstractActionInput actionInput)
{
this.actionInput = actionInput;
}
/*******************************************************************************
** Fetch a record from a table by a key field (doesn't have to be its primary key).
*******************************************************************************/
public QRecord getRecordByKey(String tableName, String keyFieldName, Serializable key) throws QException
{
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> recordMap = foreignRecordMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
if(!recordMap.containsKey(key))
{
Optional<QRecord> optRecord = GeneralProcessUtils.getRecordByField(actionInput, tableName, keyFieldName, key);
recordMap.put(key, optRecord.orElse(null));
}
return (recordMap.get(key));
}
/*******************************************************************************
** Optimization - to pre-load the records in a single query, which would otherwise
** have to be looked up one-by-one.
*******************************************************************************/
public void preloadRecords(String tableName, String keyFieldName) throws QException
{
String mapKey = tableName + "." + keyFieldName;
if(!preloadedKeys.contains(mapKey))
{
Map<Serializable, QRecord> recordMap = GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName);
foreignRecordMaps.put(mapKey, recordMap);
preloadedKeys.add(mapKey);
}
}
/*******************************************************************************
** Get a value from a record, by doing a lookup on the specified keyFieldName,
** for the specified key value.
**
*******************************************************************************/
public Serializable getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key) throws QException
{
QRecord record = getRecordByKey(tableName, keyFieldName, key);
if(record == null)
{
return (null);
}
return (record.getValue(requestedField));
}
/*******************************************************************************
** Get a value from a record, in the requested type, by doing a lookup on the
** specified keyFieldName, for the specified key value.
**
*******************************************************************************/
public <T extends Serializable> T getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key, Class<T> type) throws QException
{
Serializable value = getRecordValue(tableName, requestedField, keyFieldName, key);
return (ValueUtils.getValueAsType(type, value));
}
/*******************************************************************************
** Get the id (primary key) value from a record, by doing a lookup on the
** specified keyFieldName, for the specified key value.
**
*******************************************************************************/
public Serializable getRecordId(String tableName, String keyFieldName, Serializable key) throws QException
{
String primaryKeyField = actionInput.getInstance().getTable(tableName).getPrimaryKeyField();
return (getRecordValue(tableName, primaryKeyField, keyFieldName, key));
}
/*******************************************************************************
** Get the id (primary key) value from a record, in the requested type, by doing
** a lookup on the specified keyFieldName, for the specified key value.
**
*******************************************************************************/
public <T extends Serializable> T getRecordId(String tableName, String keyFieldName, Serializable key, Class<T> type) throws QException
{
Serializable value = getRecordId(tableName, keyFieldName, key);
return (ValueUtils.getValueAsType(type, value));
}
}

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/Pair.java
View File

@ -46,6 +46,16 @@ public class Pair<A, B> implements Cloneable
/*******************************************************************************
** static constructor (factory)
*******************************************************************************/
public static <A, B> Pair<A, B> of(A a, B b)
{
return (new Pair<>(a, b));
}
/*******************************************************************************
**
*******************************************************************************/

5
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java
View File

@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -53,7 +54,8 @@ public class QPossibleValueTranslatorTest
**
*******************************************************************************/
@BeforeEach
void beforeEach()
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
MemoryRecordStore.resetStatistics();
@ -164,6 +166,7 @@ public class QPossibleValueTranslatorTest
///////////////////////////////////////////////////////////
possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
MemoryRecordStore.setCollectStatistics(true);
MemoryRecordStore.resetStatistics();
possibleValueTranslator.translatePossibleValue(shapeField, 1);
possibleValueTranslator.translatePossibleValue(shapeField, 2);
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 queries so far");

316
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Shape.java Normal file
View File

@ -0,0 +1,316 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.data.testentities;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
/*******************************************************************************
**
*******************************************************************************/
public class Shape extends QRecordEntity
{
@QField()
private Integer id;
@QField()
private Instant createDate;
@QField()
private Instant modifyDate;
@QField()
private String name;
@QField()
private String type;
@QField()
private Integer noOfSides;
@QField()
private Boolean isPolygon;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public Shape()
{
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public Shape(QRecord record)
{
populateFromQRecord(record);
}
/*******************************************************************************
** Getter for id
**
*******************************************************************************/
public Integer getId()
{
return id;
}
/*******************************************************************************
** Setter for id
**
*******************************************************************************/
public void setId(Integer id)
{
this.id = id;
}
/*******************************************************************************
** Fluent setter for id
**
*******************************************************************************/
public Shape withId(Integer id)
{
this.id = id;
return (this);
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public Shape withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for createDate
**
*******************************************************************************/
public Instant getCreateDate()
{
return createDate;
}
/*******************************************************************************
** Setter for createDate
**
*******************************************************************************/
public void setCreateDate(Instant createDate)
{
this.createDate = createDate;
}
/*******************************************************************************
** Fluent setter for createDate
**
*******************************************************************************/
public Shape withCreateDate(Instant createDate)
{
this.createDate = createDate;
return (this);
}
/*******************************************************************************
** Getter for modifyDate
**
*******************************************************************************/
public Instant getModifyDate()
{
return modifyDate;
}
/*******************************************************************************
** Setter for modifyDate
**
*******************************************************************************/
public void setModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
}
/*******************************************************************************
** Fluent setter for modifyDate
**
*******************************************************************************/
public Shape withModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
return (this);
}
/*******************************************************************************
** Getter for type
**
*******************************************************************************/
public String getType()
{
return type;
}
/*******************************************************************************
** Setter for type
**
*******************************************************************************/
public void setType(String type)
{
this.type = type;
}
/*******************************************************************************
** Fluent setter for type
**
*******************************************************************************/
public Shape withType(String type)
{
this.type = type;
return (this);
}
/*******************************************************************************
** Getter for noOfSides
**
*******************************************************************************/
public Integer getNoOfSides()
{
return noOfSides;
}
/*******************************************************************************
** Setter for noOfSides
**
*******************************************************************************/
public void setNoOfSides(Integer noOfSides)
{
this.noOfSides = noOfSides;
}
/*******************************************************************************
** Fluent setter for noOfSides
**
*******************************************************************************/
public Shape withNoOfSides(Integer noOfSides)
{
this.noOfSides = noOfSides;
return (this);
}
/*******************************************************************************
** Getter for isPolygon
**
*******************************************************************************/
public Boolean getIsPolygon()
{
return isPolygon;
}
/*******************************************************************************
** Setter for isPolygon
**
*******************************************************************************/
public void setIsPolygon(Boolean isPolygon)
{
this.isPolygon = isPolygon;
}
/*******************************************************************************
** Fluent setter for isPolygon
**
*******************************************************************************/
public Shape withIsPolygon(Boolean isPolygon)
{
this.isPolygon = isPolygon;
return (this);
}
}

16
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java
View File

@ -55,7 +55,7 @@ class TableSyncProcessTest
**
*******************************************************************************/
@Test
void test() throws QException
void test() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
@ -86,10 +86,6 @@ class TableSyncProcessTest
qInstance.addProcess(TableSyncProcess.processMetaDataBuilder(false)
.withName(PROCESS_NAME)
.withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withDestinationTable(TABLE_NAME_PEOPLE_SYNC)
.withSourceTableKeyField("id")
.withDestinationTableForeignKeyField("sourcePersonId")
.withSyncTransformStepClass(PersonTransformClass.class)
.getProcessMetaData());
@ -142,6 +138,16 @@ class TableSyncProcessTest
return (destinationRecord);
}
@Override
protected SyncProcessConfig getSyncProcessConfig()
{
SyncProcessConfig syncProcessConfig = new SyncProcessConfig(TestUtils.TABLE_NAME_PERSON_MEMORY, "id", "peopleSync", "sourcePersonId");
syncProcessConfig.noop();
return (syncProcessConfig);
}
}
}

128
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtilsTest.java
View File

@ -30,12 +30,14 @@ import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.testentities.Shape;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
@ -46,7 +48,9 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -329,6 +333,11 @@ class GeneralProcessUtilsTest
assertEquals(3, recordMapByFirstName.size());
assertEquals(1, recordMapByFirstName.get("Darin").getValueInteger("id"));
assertEquals(3, recordMapByFirstName.get("Tim").getValueInteger("id"));
Map<String, QRecord> recordMapByFirstNameAsString = GeneralProcessUtils.loadTableToMap(queryInput, TestUtils.TABLE_NAME_PERSON_MEMORY, String.class, "firstName");
assertEquals(3, recordMapByFirstName.size());
assertEquals(1, recordMapByFirstName.get("Darin").getValueInteger("id"));
assertEquals(3, recordMapByFirstName.get("Tim").getValueInteger("id"));
}
@ -354,4 +363,123 @@ class GeneralProcessUtilsTest
assertEquals(1, map.get("Darin").size());
assertEquals(2, map.get("James").size());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetRecordByFieldOrElseThrow() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
assertNotNull(GeneralProcessUtils.getRecordByFieldOrElseThrow(new AbstractActionInput(instance, new QSession()), TestUtils.TABLE_NAME_SHAPE, "name", "Triangle"));
assertThrows(QException.class, () -> GeneralProcessUtils.getRecordByFieldOrElseThrow(new AbstractActionInput(instance, new QSession()), TestUtils.TABLE_NAME_SHAPE, "name", "notAShape"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetRecordByPrimaryKey() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
assertTrue(GeneralProcessUtils.getRecordByPrimaryKey(actionInput, TestUtils.TABLE_NAME_SHAPE, 1).isPresent());
assertFalse(GeneralProcessUtils.getRecordByPrimaryKey(actionInput, TestUtils.TABLE_NAME_SHAPE, -1).isPresent());
assertNotNull(GeneralProcessUtils.getRecordByPrimaryKeyOrElseThrow(actionInput, TestUtils.TABLE_NAME_SHAPE, 1));
assertThrows(QException.class, () -> GeneralProcessUtils.getRecordByPrimaryKeyOrElseThrow(actionInput, TestUtils.TABLE_NAME_SHAPE, -1));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCount() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
assertEquals(3, GeneralProcessUtils.count(actionInput, TestUtils.TABLE_NAME_SHAPE, null));
assertEquals(1, GeneralProcessUtils.count(actionInput, TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 2))));
assertEquals(0, GeneralProcessUtils.count(actionInput, TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IS_BLANK))));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetEntityByField() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
assertEquals("Triangle", GeneralProcessUtils.getEntityByField(actionInput, TestUtils.TABLE_NAME_SHAPE, "name", "Triangle", Shape.class).get().getName());
assertFalse(GeneralProcessUtils.getEntityByField(actionInput, TestUtils.TABLE_NAME_SHAPE, "name", "notAShape", Shape.class).isPresent());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testLoadTableAsEntities() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
List<Shape> shapes = GeneralProcessUtils.loadTable(actionInput, TestUtils.TABLE_NAME_SHAPE, Shape.class);
assertEquals(3, shapes.size());
assertTrue(shapes.get(0) instanceof Shape);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testLoadTableToMapAsEntities() throws QException
{
QInstance instance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(instance);
AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession());
Map<Serializable, Shape> map = GeneralProcessUtils.loadTableToMap(actionInput, TestUtils.TABLE_NAME_SHAPE, "id", Shape.class);
assertEquals(3, map.size());
assertTrue(map.get(1) instanceof Shape);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testRecordsToEntities() throws QException
{
List<Shape> shapes = GeneralProcessUtils.recordsToEntities(Shape.class, List.of(
new QRecord().withValue("id", 99).withValue("name", "round"),
new QRecord().withValue("id", 98).withValue("name", "flat")
));
assertEquals(2, shapes.size());
assertEquals(99, shapes.get(0).getId());
assertEquals("round", shapes.get(0).getName());
}
}

123
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java Normal file
View File

@ -0,0 +1,123 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.utils;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for RecordLookupHelper
*******************************************************************************/
class RecordLookupHelperTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
MemoryRecordStore.resetStatistics();
MemoryRecordStore.setCollectStatistics(true);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithoutPreload() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(qInstance);
RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput(qInstance, new QSession()));
MemoryRecordStore.setCollectStatistics(true);
assertEquals(2, recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "Square"));
assertEquals(2, recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "Square", Integer.class));
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
assertEquals("Circle", recordLookupHelper.getRecordValue(TestUtils.TABLE_NAME_SHAPE, "name", "id", 3));
assertEquals("Circle", recordLookupHelper.getRecordValue(TestUtils.TABLE_NAME_SHAPE, "name", "id", 3, String.class));
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
assertNull(recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "notAShape"));
assertNull(recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "notAShape", Integer.class));
assertEquals(3, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithPreload() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
TestUtils.insertDefaultShapes(qInstance);
RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput(qInstance, new QSession()));
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name");
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
assertNotNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Triangle"));
assertEquals(1, recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Triangle").getValueInteger("id"));
assertEquals("Triangle", recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Triangle").getValueString("name"));
assertEquals(2, recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Square").getValueInteger("id"));
assertEquals("Square", recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Square").getValueString("name"));
assertEquals(3, recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Circle").getValueInteger("id"));
assertEquals("Circle", recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Circle").getValueString("name"));
/////////////////////////////////////////////////////
// all those gets should run no additional queries //
/////////////////////////////////////////////////////
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
////////////////////////////////////////////////////////////////////
// make sure we don't re-do the query in a second call to preload //
////////////////////////////////////////////////////////////////////
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name");
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
///////////////////////////////////////////////////
// make sure we can preload by a different field //
///////////////////////////////////////////////////
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "id");
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
}
}