QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-37 Redo bulk processes in streamed-etl mode

Browse Source
This commit is contained in:
darin.kelkhoff
2022-09-07 16:59:42 -05:00
parent 1c75df3a09
commit b01758879c
27 changed files with 1430 additions and 1036 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
View File

@ -74,6 +74,10 @@ public class QValueFormatter
{
return formatValue(field, ValueUtils.getValueAsBigDecimal(value));
}
else if(e.getMessage().equals("f != java.lang.String"))
{
return formatValue(field, ValueUtils.getValueAsBigDecimal(value));
}
else if(e.getMessage().equals("d != java.math.BigDecimal"))
{
return formatValue(field, ValueUtils.getValueAsInteger(value));

288
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java
View File

@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
@ -62,8 +63,7 @@ public class CsvToQRecordAdapter
*******************************************************************************/
public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
{
this.recordPipe = recordPipe;
doBuildRecordsFromCsv(csv, table, mapping, recordCustomizer);
buildRecordsFromCsv(new InputWrapper().withRecordPipe(recordPipe).withCsv(csv).withTable(table).withMapping(mapping).withRecordCustomizer(recordCustomizer));
}
@ -75,8 +75,7 @@ public class CsvToQRecordAdapter
*******************************************************************************/
public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
{
this.recordList = new ArrayList<>();
doBuildRecordsFromCsv(csv, table, mapping, null);
buildRecordsFromCsv(new InputWrapper().withCsv(csv).withTable(table).withMapping(mapping));
return (recordList);
}
@ -88,13 +87,29 @@ public class CsvToQRecordAdapter
**
** todo - meta-data validation, type handling
*******************************************************************************/
public void doBuildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
public void buildRecordsFromCsv(InputWrapper inputWrapper)
{
String csv = inputWrapper.getCsv();
AbstractQFieldMapping<?> mapping = inputWrapper.getMapping();
Consumer<QRecord> recordCustomizer = inputWrapper.getRecordCustomizer();
QTableMetaData table = inputWrapper.getTable();
Integer limit = inputWrapper.getLimit();
if(!StringUtils.hasContent(csv))
{
throw (new IllegalArgumentException("Empty csv value was provided."));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// if caller supplied a record pipe, use it -- but if it's null, then create a recordList to populate. //
// see addRecord method for usage. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
this.recordPipe = inputWrapper.getRecordPipe();
if(this.recordPipe == null)
{
this.recordList = new ArrayList<>();
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// once, from a DOS csv file (that had come from Excel), we had a "" character (FEFF, Byte-order marker) at the start of a //
// CSV, which caused our first header to not match... So, let us strip away any FEFF or FFFE's at the start of CSV strings. //
@ -120,9 +135,12 @@ public class CsvToQRecordAdapter
List<String> headers = csvParser.getHeaderNames();
headers = makeHeadersUnique(headers);
List<CSVRecord> csvRecords = csvParser.getRecords();
for(CSVRecord csvRecord : csvRecords)
Iterator<CSVRecord> csvIterator = csvParser.iterator();
int recordCount = 0;
while(csvIterator.hasNext())
{
CSVRecord csvRecord = csvIterator.next();
//////////////////////////////////////////////////////////////////
// put values from the CSV record into a map of header -> value //
//////////////////////////////////////////////////////////////////
@ -144,6 +162,12 @@ public class CsvToQRecordAdapter
runRecordCustomizer(recordCustomizer, qRecord);
addRecord(qRecord);
recordCount++;
if(limit != null && recordCount > limit)
{
break;
}
}
}
else if(AbstractQFieldMapping.SourceType.INDEX.equals(mapping.getSourceType()))
@ -155,9 +179,12 @@ public class CsvToQRecordAdapter
CSVFormat.DEFAULT
.withTrim());
List<CSVRecord> csvRecords = csvParser.getRecords();
for(CSVRecord csvRecord : csvRecords)
Iterator<CSVRecord> csvIterator = csvParser.iterator();
int recordCount = 0;
while(csvIterator.hasNext())
{
CSVRecord csvRecord = csvIterator.next();
/////////////////////////////////////////////////////////////////
// put values from the CSV record into a map of index -> value //
/////////////////////////////////////////////////////////////////
@ -180,6 +207,12 @@ public class CsvToQRecordAdapter
runRecordCustomizer(recordCustomizer, qRecord);
addRecord(qRecord);
recordCount++;
if(limit != null && recordCount > limit)
{
break;
}
}
}
else
@ -261,4 +294,241 @@ public class CsvToQRecordAdapter
}
}
/*******************************************************************************
** Getter for recordList - note - only is valid if you don't supply a pipe in
** the input. If you do supply a pipe, then you get an exception if you call here!
**
*******************************************************************************/
public List<QRecord> getRecordList()
{
if(recordPipe != null)
{
throw (new IllegalStateException("getRecordList called on a CSVToQRecordAdapter that ran with a recordPipe."));
}
return recordList;
}
/*******************************************************************************
**
*******************************************************************************/
public static class InputWrapper
{
private RecordPipe recordPipe;
private String csv;
private QTableMetaData table;
private AbstractQFieldMapping<?> mapping;
private Consumer<QRecord> recordCustomizer;
private Integer limit;
/*******************************************************************************
** Getter for recordPipe
**
*******************************************************************************/
public RecordPipe getRecordPipe()
{
return recordPipe;
}
/*******************************************************************************
** Setter for recordPipe
**
*******************************************************************************/
public void setRecordPipe(RecordPipe recordPipe)
{
this.recordPipe = recordPipe;
}
/*******************************************************************************
** Fluent setter for recordPipe
**
*******************************************************************************/
public InputWrapper withRecordPipe(RecordPipe recordPipe)
{
this.recordPipe = recordPipe;
return (this);
}
/*******************************************************************************
** Getter for csv
**
*******************************************************************************/
public String getCsv()
{
return csv;
}
/*******************************************************************************
** Setter for csv
**
*******************************************************************************/
public void setCsv(String csv)
{
this.csv = csv;
}
/*******************************************************************************
** Fluent setter for csv
**
*******************************************************************************/
public InputWrapper withCsv(String csv)
{
this.csv = csv;
return (this);
}
/*******************************************************************************
** Getter for table
**
*******************************************************************************/
public QTableMetaData getTable()
{
return table;
}
/*******************************************************************************
** Setter for table
**
*******************************************************************************/
public void setTable(QTableMetaData table)
{
this.table = table;
}
/*******************************************************************************
** Fluent setter for table
**
*******************************************************************************/
public InputWrapper withTable(QTableMetaData table)
{
this.table = table;
return (this);
}
/*******************************************************************************
** Getter for mapping
**
*******************************************************************************/
public AbstractQFieldMapping<?> getMapping()
{
return mapping;
}
/*******************************************************************************
** Setter for mapping
**
*******************************************************************************/
public void setMapping(AbstractQFieldMapping<?> mapping)
{
this.mapping = mapping;
}
/*******************************************************************************
** Fluent setter for mapping
**
*******************************************************************************/
public InputWrapper withMapping(AbstractQFieldMapping<?> mapping)
{
this.mapping = mapping;
return (this);
}
/*******************************************************************************
** Getter for recordCustomizer
**
*******************************************************************************/
public Consumer<QRecord> getRecordCustomizer()
{
return recordCustomizer;
}
/*******************************************************************************
** Setter for recordCustomizer
**
*******************************************************************************/
public void setRecordCustomizer(Consumer<QRecord> recordCustomizer)
{
this.recordCustomizer = recordCustomizer;
}
/*******************************************************************************
** Fluent setter for recordCustomizer
**
*******************************************************************************/
public InputWrapper withRecordCustomizer(Consumer<QRecord> recordCustomizer)
{
this.recordCustomizer = recordCustomizer;
return (this);
}
/*******************************************************************************
** Getter for limit
**
*******************************************************************************/
public Integer getLimit()
{
return limit;
}
/*******************************************************************************
** Setter for limit
**
*******************************************************************************/
public void setLimit(Integer limit)
{
this.limit = limit;
}
/*******************************************************************************
** Fluent setter for limit
**
*******************************************************************************/
public InputWrapper withLimit(Integer limit)
{
this.limit = limit;
return (this);
}
}
}

201
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -22,38 +22,39 @@
package com.kingsrook.qqq.backend.core.instances;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteStoreStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditReceiveValuesStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditStoreRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertStoreRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.logging.log4j.LogManager;
@ -293,6 +294,20 @@ public class QInstanceEnricher
*******************************************************************************/
private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName)
{
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
BulkInsertExtractStep.class,
BulkInsertTransformStep.class,
LoadViaInsertStep.class,
values
)
.withName(processName)
.withLabel(table.getLabel() + " Bulk Insert")
.withTableName(table.getName())
.withIsHidden(true);
List<QFieldMetaData> editableFields = table.getFields().values().stream()
.filter(QFieldMetaData::getIsEditable)
.toList();
@ -307,54 +322,13 @@ public class QInstanceEnricher
.withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "Upload a CSV file with the following columns: " + fieldsForHelpText))
.withValue("previewText", "file upload instructions")
.withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM));
QBackendStepMetaData receiveFileStep = new QBackendStepMetaData()
.withName("receiveFile")
.withCode(new QCodeReference(BulkInsertReceiveFileStep.class))
.withOutputMetaData(new QFunctionOutputMetaData()
.withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData()
.withName("review")
.withRecordListFields(editableFields)
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below were parsed from your file, and will be inserted if you click Submit."))
.withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows"))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST));
QBackendStepMetaData storeStep = new QBackendStepMetaData()
.withName("storeRecords")
.withCode(new QCodeReference(BulkInsertStoreRecordsStep.class))
.withOutputMetaData(new QFunctionOutputMetaData()
.withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData()
.withName("results")
.withRecordListFields(new ArrayList<>(table.getFields().values()))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below have been inserted."))
.withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows"))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST));
qInstance.addProcess(
new QProcessMetaData()
.withName(processName)
.withLabel(table.getLabel() + " Bulk Insert")
.withTableName(table.getName())
.withIsHidden(true)
.withStepList(List.of(
uploadScreen,
receiveFileStep,
reviewScreen,
storeStep,
resultsScreen
)));
process.addStep(0, uploadScreen);
process.getFrontendStep("review").setRecordListFields(editableFields);
qInstance.addProcess(process);
}
@ -364,6 +338,22 @@ public class QInstanceEnricher
*******************************************************************************/
private void defineTableBulkEdit(QInstance qInstance, QTableMetaData table, String processName)
{
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table.getName());
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE);
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
ExtractViaQueryStep.class,
BulkEditTransformStep.class,
LoadViaUpdateStep.class,
values
)
.withName(processName)
.withLabel(table.getLabel() + " Bulk Edit")
.withTableName(table.getName())
.withIsHidden(true);
List<QFieldMetaData> editableFields = table.getFields().values().stream()
.filter(QFieldMetaData::getIsEditable)
.toList();
@ -381,54 +371,9 @@ public class QInstanceEnricher
Fields whose switches are off will not be updated."""))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_EDIT_FORM));
QBackendStepMetaData receiveValuesStep = new QBackendStepMetaData()
.withName("receiveValues")
.withCode(new QCodeReference(BulkEditReceiveValuesStep.class))
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName(table.getName()))
.withField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, QFieldType.STRING))
.withFields(editableFields));
QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData()
.withName("review")
.withRecordListFields(editableFields)
.withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below will be updated if you click Submit."))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST));
QBackendStepMetaData storeStep = new QBackendStepMetaData()
.withName("storeRecords")
.withCode(new QCodeReference(BulkEditStoreRecordsStep.class))
.withOutputMetaData(new QFunctionOutputMetaData()
.withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData()
.withName("results")
.withRecordListFields(new ArrayList<>(table.getFields().values()))
.withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below have been updated."))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST));
qInstance.addProcess(
new QProcessMetaData()
.withName(processName)
.withLabel(table.getLabel() + " Bulk Edit")
.withTableName(table.getName())
.withIsHidden(true)
.withStepList(List.of(
LoadInitialRecordsStep.defineMetaData(table.getName()),
editScreen,
receiveValuesStep,
reviewScreen,
storeStep,
resultsScreen
)));
process.addStep(0, editScreen);
process.getFrontendStep("review").setRecordListFields(editableFields);
qInstance.addProcess(process);
}
@ -438,38 +383,26 @@ public class QInstanceEnricher
*******************************************************************************/
private void defineTableBulkDelete(QInstance qInstance, QTableMetaData table, String processName)
{
QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData()
.withName("review")
.withRecordListFields(new ArrayList<>(table.getFields().values()))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below will be deleted if you click Submit."))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST));
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table.getName());
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_DELETE);
QBackendStepMetaData storeStep = new QBackendStepMetaData()
.withName("delete")
.withCode(new QCodeReference(BulkDeleteStoreStep.class));
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
ExtractViaQueryStep.class,
BulkDeleteTransformStep.class,
LoadViaDeleteStep.class,
values
)
.withName(processName)
.withLabel(table.getLabel() + " Bulk Delete")
.withTableName(table.getName())
.withIsHidden(true);
QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData()
.withName("results")
.withRecordListFields(new ArrayList<>(table.getFields().values()))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below have been deleted."))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST));
List<QFieldMetaData> tableFields = table.getFields().values().stream().toList();
process.getFrontendStep("review").setRecordListFields(tableFields);
qInstance.addProcess(
new QProcessMetaData()
.withName(processName)
.withLabel(table.getLabel() + " Bulk Delete")
.withTableName(table.getName())
.withIsHidden(true)
.withStepList(List.of(
LoadInitialRecordsStep.defineMetaData(table.getName()),
reviewScreen,
storeStep,
resultsScreen
)));
qInstance.addProcess(process);
}

25
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java
View File

@ -35,7 +35,7 @@ import java.util.List;
public class ProcessSummaryLine implements Serializable
{
private Status status;
private Integer count;
private Integer count = 0;
private String message;
//////////////////////////////////////////////////////////////////////////
@ -77,7 +77,16 @@ public class ProcessSummaryLine implements Serializable
{
this.status = status;
this.message = message;
this.count = 0;
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLine(Status status)
{
this.status = status;
}
@ -185,12 +194,22 @@ public class ProcessSummaryLine implements Serializable
**
*******************************************************************************/
public void incrementCount()
{
incrementCount(1);
}
/*******************************************************************************
**
*******************************************************************************/
public void incrementCount(int amount)
{
if(count == null)
{
count = 0;
}
count++;
count += amount;
}

42
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.delete;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -35,8 +36,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
*******************************************************************************/
public class DeleteInput extends AbstractTableActionInput
{
private List<Serializable> primaryKeys;
private QQueryFilter queryFilter;
private QBackendTransaction transaction;
private List<Serializable> primaryKeys;
private QQueryFilter queryFilter;
@ -59,6 +61,40 @@ public class DeleteInput extends AbstractTableActionInput
/*******************************************************************************
** Getter for transaction
**
*******************************************************************************/
public QBackendTransaction getTransaction()
{
return transaction;
}
/*******************************************************************************
** Setter for transaction
**
*******************************************************************************/
public void setTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
}
/*******************************************************************************
** Fluent setter for transaction
**
*******************************************************************************/
public DeleteInput withTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
return (this);
}
/*******************************************************************************
** Getter for ids
**
@ -92,6 +128,7 @@ public class DeleteInput extends AbstractTableActionInput
}
/*******************************************************************************
** Getter for queryFilter
**
@ -113,6 +150,7 @@ public class DeleteInput extends AbstractTableActionInput
}
/*******************************************************************************
** Fluent setter for queryFilter
**

101
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java
View File

@ -1,101 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import java.io.IOException;
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Backend step to do a bulk delete.
*******************************************************************************/
public class BulkDeleteStoreStep implements BackendStep
{
public static final String ERROR_COUNT = "errorCount";
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records...");
runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal();
DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance());
deleteInput.setSession(runBackendStepInput.getSession());
deleteInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
deleteInput.setTableName(runBackendStepInput.getTableName());
String queryFilterJSON = runBackendStepInput.getValueString("queryFilterJSON");
if(StringUtils.hasContent(queryFilterJSON))
{
try
{
deleteInput.setQueryFilter(JsonUtils.toObject(queryFilterJSON, QQueryFilter.class));
}
catch(IOException e)
{
throw (new QException("Error loading record query filter from process", e));
}
}
else if(CollectionUtils.nullSafeHasContents(runBackendStepInput.getRecords()))
{
String primaryKeyField = runBackendStepInput.getTable().getPrimaryKeyField();
List<Serializable> primaryKeyList = runBackendStepInput.getRecords().stream()
.map(r -> r.getValue(primaryKeyField))
.toList();
deleteInput.setPrimaryKeys(primaryKeyList);
}
else
{
throw (new QException("Missing required inputs (queryFilterJSON or record list)"));
}
DeleteAction deleteAction = new DeleteAction();
DeleteOutput deleteOutput = deleteAction.execute(deleteInput);
List<QRecord> recordsWithErrors = Objects.requireNonNullElse(deleteOutput.getRecordsWithErrors(), Collections.emptyList());
runBackendStepOutput.addValue(ERROR_COUNT, recordsWithErrors.size());
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
}
}

126
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java Normal file
View File

@ -0,0 +1,126 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
/*******************************************************************************
** Transform step for generic table bulk-insert ETL process
*******************************************************************************/
public class BulkDeleteTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private String tableLabel;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
///////////////////////////////////////////////////////
// capture the table label - for the process summary //
///////////////////////////////////////////////////////
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
if(table != null)
{
tableLabel = table.getLabel();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// on the validate step, we haven't read the full file, so we don't know how many rows there are - thus //
// record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
{
if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record " + "%,d".formatted(okSummary.getCount()));
}
else
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record");
}
}
else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE))
{
if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " record " + "%,d".formatted(okSummary.getCount()));
}
else
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " records");
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// no transformation needs done - just pass records through from input to output, and assume all are OK //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
okSummary.incrementCount(runBackendStepInput.getRecords().size());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
{
if(isForResultScreen)
{
okSummary.setMessage(tableLabel + " records were deleted.");
}
else
{
okSummary.setMessage(tableLabel + " records will be deleted.");
}
ArrayList<ProcessSummaryLine> rs = new ArrayList<>();
rs.add(okSummary);
return (rs);
}
}

70
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java
View File

@ -1,70 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Backend step to receive values for a bulk edit.
*******************************************************************************/
public class BulkEditReceiveValuesStep implements BackendStep
{
public static final String FIELD_ENABLED_FIELDS = "bulkEditEnabledFields";
public static final String FIELD_VALUES_BEING_UPDATED = "valuesBeingUpdated";
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
String enabledFieldsString = runBackendStepInput.getValueString(FIELD_ENABLED_FIELDS);
String[] enabledFields = enabledFieldsString.split(",");
////////////////////////////////////////////////////////////////////////////////////////////
// put the value in all the records (note, this is just for display on the review screen, //
// and/or if we wanted to do some validation - this is NOT what will be store, as the //
// Update action only wants fields that are being changed. //
////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
{
for(String fieldName : enabledFields)
{
Serializable value = runBackendStepInput.getValue(fieldName);
record.setValue(fieldName, value);
}
}
BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "will be");
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
}
}

90
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java
View File

@ -1,90 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Backend step to store the records from a bulk insert file
*******************************************************************************/
public class BulkEditStoreRecordsStep implements BackendStep
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
String enabledFieldsString = runBackendStepInput.getValueString(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS);
String[] enabledFields = enabledFieldsString.split(",");
QTableMetaData table = runBackendStepInput.getTable();
List<QRecord> recordsToUpdate = new ArrayList<>();
runBackendStepInput.getAsyncJobCallback().updateStatus("Updating values in records...");
int i = 1;
for(QRecord record : runBackendStepInput.getRecords())
{
runBackendStepInput.getAsyncJobCallback().updateStatus(i++, runBackendStepInput.getRecords().size());
QRecord recordToUpdate = new QRecord();
recordsToUpdate.add(recordToUpdate);
recordToUpdate.setValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField()));
for(String fieldName : enabledFields)
{
Serializable value = runBackendStepInput.getValue(fieldName);
recordToUpdate.setValue(fieldName, value);
}
}
runBackendStepInput.getAsyncJobCallback().updateStatus("Storing updated records...");
runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal();
UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance());
updateInput.setSession(runBackendStepInput.getSession());
updateInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
updateInput.setAreAllValuesBeingUpdatedTheSame(true);
updateInput.setTableName(runBackendStepInput.getTableName());
updateInput.setRecords(recordsToUpdate);
UpdateAction updateAction = new UpdateAction();
UpdateOutput updateOutput = updateAction.execute(updateInput);
runBackendStepOutput.setRecords(updateOutput.getRecords());
BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "was");
}
}

215
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java Normal file
View File

@ -0,0 +1,215 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** Transform step for generic table bulk-edit ETL process
*******************************************************************************/
public class BulkEditTransformStep extends AbstractTransformStep
{
public static final String FIELD_ENABLED_FIELDS = "bulkEditEnabledFields";
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private List<ProcessSummaryLine> infoSummaries = new ArrayList<>();
private QTableMetaData table;
private String tableLabel;
private String[] enabledFields;
private boolean isValidateStep;
private boolean isExecuteStep;
private boolean haveRecordCount;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
///////////////////////////////////////////////////////
// capture the table label - for the process summary //
///////////////////////////////////////////////////////
table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
if(table != null)
{
tableLabel = table.getLabel();
}
String enabledFieldsString = runBackendStepInput.getValueString(FIELD_ENABLED_FIELDS);
enabledFields = enabledFieldsString.split(",");
isValidateStep = runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
isExecuteStep = runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE);
haveRecordCount = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) != null;
buildInfoSummaryLines(runBackendStepInput, enabledFields);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// on the validate step, we haven't read the full file, so we don't know how many rows there are - thus //
// record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(isValidateStep)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records");
if(!haveRecordCount)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing record " + "%,d".formatted(okSummary.getCount()));
}
}
else if(isExecuteStep)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Editing " + tableLabel + " records");
if(!haveRecordCount)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Editing " + tableLabel + " record " + "%,d".formatted(okSummary.getCount()));
}
}
List<QRecord> outputRecords = new ArrayList<>();
if(isExecuteStep)
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// for the execute step - create new record objects, just with the primary key, and the fields being updated. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
{
QRecord recordToUpdate = new QRecord();
recordToUpdate.setValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField()));
outputRecords.add(recordToUpdate);
setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, recordToUpdate);
}
}
else
{
////////////////////////////////////////////////////////////////////////////////////////////
// put the value in all the records (note, this is just for display on the review screen, //
// and/or if we wanted to do some validation - this is NOT what will be store, as the //
// Update action only wants fields that are being changed. //
////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
{
outputRecords.add(record);
setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, record);
}
}
runBackendStepOutput.setRecords(outputRecords);
okSummary.incrementCount(runBackendStepInput.getRecords().size());
}
/*******************************************************************************
**
*******************************************************************************/
private void buildInfoSummaryLines(RunBackendStepInput runBackendStepInput, String[] enabledFields)
{
QValueFormatter qValueFormatter = new QValueFormatter();
for(String fieldName : enabledFields)
{
QFieldMetaData field = table.getField(fieldName);
String label = field.getLabel();
Serializable value = runBackendStepInput.getValue(fieldName);
ProcessSummaryLine summaryLine = new ProcessSummaryLine(Status.INFO);
summaryLine.setCount(null);
infoSummaries.add(summaryLine);
String verb = isExecuteStep ? "was" : "will be";
if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
{
String formattedValue = qValueFormatter.formatValue(field, value); // todo - PVS!
summaryLine.setMessage(label + " " + verb + " set to: " + formattedValue);
}
else
{
summaryLine.setMessage(label + " " + verb + " cleared out");
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private void setUpdatedFieldsInRecord(RunBackendStepInput runBackendStepInput, String[] enabledFields, QRecord record)
{
for(String fieldName : enabledFields)
{
Serializable value = runBackendStepInput.getValue(fieldName);
record.setValue(fieldName, value);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
{
if(isForResultScreen)
{
okSummary.setMessage(tableLabel + " records were edited.");
}
else
{
okSummary.setMessage(tableLabel + " records will be edited.");
}
ArrayList<ProcessSummaryLine> rs = new ArrayList<>();
rs.add(okSummary);
rs.addAll(infoSummaries);
return (rs);
}
}

68
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java
View File

@ -1,68 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** Utility methods used for Bulk Edit steps
*******************************************************************************/
public class BulkEditUtils
{
/*******************************************************************************
**
*******************************************************************************/
public static void setFieldValuesBeingUpdated(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, String[] enabledFields, String verb)
{
/////////////////////////////////////////////////////////////////////
// build the string to show the user what fields are being changed //
/////////////////////////////////////////////////////////////////////
List<String> valuesBeingUpdated = new ArrayList<>();
QTableMetaData table = runBackendStepInput.getTable();
for(String fieldName : enabledFields)
{
String label = table.getField(fieldName).getLabel();
Serializable value = runBackendStepInput.getValue(fieldName);
if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
{
valuesBeingUpdated.add(label + " " + verb + " set to: " + value);
}
else
{
valuesBeingUpdated.add(label + " " + verb + " cleared out");
}
}
runBackendStepOutput.addValue(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated));
}
}

57
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java
View File

@ -31,23 +31,22 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
import com.kingsrook.qqq.backend.core.state.AbstractStateKey;
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
/*******************************************************************************
** Utility methods used by bulk insert steps
** Extract step for generic table bulk-insert ETL process
*******************************************************************************/
public class BulkInsertUtils
public class BulkInsertExtractStep extends AbstractExtractStep
{
/*******************************************************************************
**
*******************************************************************************/
static List<QRecord> getQRecordsFromFile(RunBackendStepInput runBackendStepInput) throws QException
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME);
Optional<QUploadedFile> optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey);
@ -70,33 +69,35 @@ public class BulkInsertUtils
mapping.addMapping(entry.getKey(), entry.getValue().getLabel());
}
List<QRecord> qRecords;
//////////////////////////////////////////////////////////////////////////
// get the non-editable fields - they'll be blanked out in a customizer //
//////////////////////////////////////////////////////////////////////////
List<QFieldMetaData> nonEditableFields = table.getFields().values().stream()
.filter(f -> !f.getIsEditable())
.toList();
if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv"))
{
qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping);
new CsvToQRecordAdapter().buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
.withRecordPipe(getRecordPipe())
.withLimit(getLimit())
.withCsv(new String(bytes))
.withTable(runBackendStepInput.getInstance().getTable(tableName))
.withMapping(mapping)
.withRecordCustomizer((record) ->
{
////////////////////////////////////////////
// remove values from non-editable fields //
////////////////////////////////////////////
for(QFieldMetaData nonEditableField : nonEditableFields)
{
record.setValue(nonEditableField.getName(), null);
}
}));
}
else
{
throw (new QUserFacingException("Unsupported file type."));
}
////////////////////////////////////////////////
// remove values from any non-editable fields //
////////////////////////////////////////////////
List<QFieldMetaData> nonEditableFields = table.getFields().values().stream()
.filter(f -> !f.getIsEditable())
.toList();
if(!nonEditableFields.isEmpty())
{
for(QRecord qRecord : qRecords)
{
for(QFieldMetaData nonEditableField : nonEditableFields)
{
qRecord.setValue(nonEditableField.getName(), null);
}
}
}
return (qRecords);
}
}

49
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java
View File

@ -1,49 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Backend step to receive a bulk-insert upload file
*******************************************************************************/
public class BulkInsertReceiveFileStep implements BackendStep
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
List<QRecord> qRecords = BulkInsertUtils.getQRecordsFromFile(runBackendStepInput);
runBackendStepOutput.addValue("noOfFileRows", qRecords.size());
runBackendStepOutput.setRecords(qRecords);
}
}

119
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java Normal file
View File

@ -0,0 +1,119 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
/*******************************************************************************
** Transform step for generic table bulk-insert ETL process
*******************************************************************************/
public class BulkInsertTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private String tableLabel;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
///////////////////////////////////////////////////////
// capture the table label - for the process summary //
///////////////////////////////////////////////////////
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
if(table != null)
{
tableLabel = table.getLabel();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// on the validate step, we haven't read the full file, so we don't know how many rows there are - thus //
// record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(okSummary.getCount()));
}
else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE))
{
if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " record " + "%,d".formatted(okSummary.getCount()));
}
else
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " records");
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// no transformation needs done - just pass records through from input to output, and assume all are OK //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
okSummary.incrementCount(runBackendStepInput.getRecords().size());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
{
if(isForResultScreen)
{
okSummary.setMessage(tableLabel + " records were inserted.");
}
else
{
okSummary.setMessage(tableLabel + " records will be inserted.");
}
ArrayList<ProcessSummaryLine> rs = new ArrayList<>();
rs.add(okSummary);
return (rs);
}
}

53
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java
View File

@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit
import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -31,9 +33,13 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
@ -94,10 +100,12 @@ public class ExtractViaQueryStep extends AbstractExtractStep
*******************************************************************************/
protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException
{
String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson");
Serializable defaultQueryFilter = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER);
//////////////////////////////////////////////////////////////////////////////////////
// if the queryFilterJson field is populated, read the filter from it and return it //
//////////////////////////////////////////////////////////////////////////////////////
String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson");
if(queryFilterJson != null)
{
return getQueryFilterFromJson(queryFilterJson, "Error loading query filter from json field");
@ -111,20 +119,41 @@ public class ExtractViaQueryStep extends AbstractExtractStep
runBackendStepInput.addValue("queryFilterJson", JsonUtils.toJson(queryFilter));
return (queryFilter);
}
else
else if(defaultQueryFilter instanceof QQueryFilter filter)
{
/////////////////////////////////////////////////////
// else, see if a defaultQueryFilter was specified //
/////////////////////////////////////////////////////
Serializable defaultQueryFilter = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER);
if(defaultQueryFilter instanceof QQueryFilter filter)
/////////////////////////////////////////////////////////////////////////////
// else, see if a defaultQueryFilter was specified as a QueryFilter object //
/////////////////////////////////////////////////////////////////////////////
return (filter);
}
else if(defaultQueryFilter instanceof String string)
{
/////////////////////////////////////////////////////////////////////////////
// else, see if a defaultQueryFilter was specified as a JSON string
/////////////////////////////////////////////////////////////////////////////
return getQueryFilterFromJson(string, "Error loading default query filter from json");
}
else if(StringUtils.hasContent(runBackendStepInput.getValueString("filterJSON")))
{
///////////////////////////////////////////////////////////////////////
// else, check for filterJSON from a frontend launching of a process //
///////////////////////////////////////////////////////////////////////
return getQueryFilterFromJson(runBackendStepInput.getValueString("filterJSON"), "Error loading default query filter from json");
}
else if(StringUtils.hasContent(runBackendStepInput.getValueString("recordIds")))
{
//////////////////////////////////////////////////////////////////////
// else, check for recordIds from a frontend launching of a process //
//////////////////////////////////////////////////////////////////////
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
if(table == null)
{
return (filter);
}
else if(defaultQueryFilter instanceof String string)
{
return getQueryFilterFromJson(string, "Error loading default query filter from json");
throw (new QException("source table name was not set - could not load records by id"));
}
String recordIds = runBackendStepInput.getValueString("recordIds");
Serializable[] split = recordIds.split(",");
List<Serializable> idStrings = Arrays.stream(split).toList();
return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, idStrings)));
}
throw (new QException("Could not find query filter for Extract step."));

49
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStep.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java
View File

@ -19,41 +19,62 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import java.util.Optional;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Backend step to store the records from a bulk insert file
** Generic implementation of a LoadStep - that runs a Delete action for a
** specified table.
*******************************************************************************/
public class BulkInsertStoreRecordsStep implements BackendStep
public class LoadViaDeleteStep extends AbstractLoadStep
{
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
List<QRecord> qRecords = BulkInsertUtils.getQRecordsFromFile(runBackendStepInput);
QTableMetaData table = runBackendStepInput.getTable();
DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance());
deleteInput.setSession(runBackendStepInput.getSession());
deleteInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
deleteInput.setPrimaryKeys(runBackendStepInput.getRecords().stream().map(r -> r.getValue(table.getPrimaryKeyField())).collect(Collectors.toList()));
// todo? can make more efficient deletes, maybe? deleteInput.setQueryFilter();
getTransaction().ifPresent(deleteInput::setTransaction);
new DeleteAction().execute(deleteInput);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance());
insertInput.setSession(runBackendStepInput.getSession());
insertInput.setTableName(runBackendStepInput.getTableName());
insertInput.setRecords(qRecords);
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
InsertAction insertAction = new InsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
runBackendStepOutput.setRecords(insertOutput.getRecords());
return (Optional.of(new InsertAction().openTransaction(insertInput)));
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java
View File

@ -82,6 +82,7 @@ public class StreamedETLWithFrontendProcess
public static final String DEFAULT_PREVIEW_MESSAGE_FOR_INSERT = "This is a preview of the records that will be created.";
public static final String DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE = "This is a preview of the records that will be updated.";
public static final String DEFAULT_PREVIEW_MESSAGE_FOR_DELETE = "This is a preview of the records that will be deleted.";
public static final String FIELD_PREVIEW_MESSAGE = "previewMessage";

74
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java
View File

@ -1,74 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for BulkDeleteStoreStep
*******************************************************************************/
class BulkDeleteStoreStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithoutFilter() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.setRecords(TestUtils.queryTable(TestUtils.defineTablePerson().getName()));
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkDeleteStoreStep().run(stepInput, stepOutput);
assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithFilter() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue("queryFilterJSON", JsonUtils.toJson(new QQueryFilter()));
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkDeleteStoreStep().run(stepInput, stepOutput);
assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
}
}

119
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTest.java Normal file
View File

@ -0,0 +1,119 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for full bulk edit process
*******************************************************************************/
class BulkDeleteTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
//////////////////////////////
// insert some test records //
//////////////////////////////
QInstance qInstance = TestUtils.defineInstance();
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of(
new QRecord().withValue("id", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("email", "darin.kelkhoff@kingsrook.com"),
new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("email", "tim.chamberlain@kingsrook.com"),
new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("lastName", "Maes").withValue("email", "james.maes@kingsrook.com")
));
//////////////////////////////////
// set up the run-process input //
//////////////////////////////////
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(TestUtils.getMockSession());
runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkDelete");
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER,
new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 3))));
RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
String processUUID = runProcessOutput.getProcessUUID();
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
runProcessInput.addValue("processUUID", processUUID);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
assertThat(runProcessOutput.getException()).isEmpty();
@SuppressWarnings("unchecked")
List<ProcessSummaryLine> processSummaryLines = (List<ProcessSummaryLine>) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
assertThat(processSummaryLines).hasSize(1);
assertThat(processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.OK))).hasSize(1);
//////////////////////////////////////////////////////////////
// query for the records - assert that only id=2 exists now //
//////////////////////////////////////////////////////////////
List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
assertEquals(2, records.get(0).getValueInteger("id"));
}
}

78
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStepTest.java
View File

@ -1,78 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for BulkEditReceiveValuesStep
*******************************************************************************/
class BulkEditReceiveValuesStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate");
stepInput.addValue("firstName", "Johnny");
stepInput.addValue("email", null);
stepInput.addValue("birthDate", "1909-01-09");
List<QRecord> records = TestUtils.queryTable(TestUtils.defineTablePerson().getName());
stepInput.setRecords(records);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkEditReceiveValuesStep().run(stepInput, stepOutput);
String valuesBeingUpdated = stepOutput.getValueString(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED);
assertThat(valuesBeingUpdated).matches("(?s).*First Name.*Johnny.*");
assertThat(valuesBeingUpdated).matches("(?s).*Email will be cleared.*");
assertThat(valuesBeingUpdated).matches("(?s).*Birth Date.*1909-01-09.*");
int count = 0;
for(QRecord record : stepOutput.getRecords())
{
assertEquals("Johnny", record.getValueString("firstName"));
assertNull(record.getValue("email"));
// todo value utils needed in getValueDate... assertEquals(LocalDate.of(1909, 1, 9), record.getValueDate("birthDate"));
count++;
}
assertEquals(records.size(), count);
}
}

85
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStepTest.java
View File

@ -1,85 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for BulkEditStoreRecordsStep
*******************************************************************************/
class BulkEditStoreRecordsStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate");
stepInput.addValue("firstName", "Johnny");
stepInput.addValue("email", null);
stepInput.addValue("birthDate", "1909-01-09");
List<QRecord> records = TestUtils.queryTable(TestUtils.defineTablePerson().getName());
stepInput.setRecords(records);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkEditStoreRecordsStep().run(stepInput, stepOutput);
assertRecordValues(stepOutput.getRecords());
// re-fetch the records, make sure they are updated.
// but since Mock backend doesn't actually update them, we can't do this..
// todo - implement an in-memory backend, that would do this.
// List<QRecord> updatedRecords = TestUtils.queryTable(TestUtils.defineTablePerson().getName());
// assertRecordValues(updatedRecords);
}
/*******************************************************************************
**
*******************************************************************************/
private void assertRecordValues(List<QRecord> records)
{
for(QRecord record : records)
{
assertEquals("Johnny", record.getValueString("firstName"));
assertNull(record.getValue("email"));
// todo value utils needed in getValueDate... assertEquals(LocalDate.of(1909, 1, 9), record.getValueDate("birthDate"));
}
}
}

145
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java Normal file
View File

@ -0,0 +1,145 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.util.List;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for full bulk edit process
*******************************************************************************/
class BulkEditTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
//////////////////////////////
// insert some test records //
//////////////////////////////
QInstance qInstance = TestUtils.defineInstance();
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of(
new QRecord().withValue("id", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("email", "darin.kelkhoff@kingsrook.com"),
new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("email", "tim.chamberlain@kingsrook.com"),
new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("lastName", "Maes").withValue("email", "james.maes@kingsrook.com")
));
//////////////////////////////////
// set up the run-process input //
//////////////////////////////////
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(TestUtils.getMockSession());
runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkEdit");
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER,
new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2))));
RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
String processUUID = runProcessOutput.getProcessUUID();
runProcessInput.addValue(BulkEditTransformStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate");
runProcessInput.addValue("firstName", "Johnny");
runProcessInput.addValue("email", null);
runProcessInput.addValue("birthDate", "1909-01-09");
runProcessInput.setProcessUUID(processUUID);
runProcessInput.setStartAfterStep("edit");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
assertThat(runProcessOutput.getException()).isEmpty();
@SuppressWarnings("unchecked")
List<ProcessSummaryLine> processSummaryLines = (List<ProcessSummaryLine>) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
assertThat(processSummaryLines).hasSize(4);
assertThat(processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.OK))).hasSize(1);
List<ProcessSummaryLine> infoLines = processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.INFO)).collect(Collectors.toList());
assertThat(infoLines).hasSize(3);
assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*First Name.*Johnny.*"));
assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*Email was cleared.*"));
assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*Birth Date.*1909-01-09.*"));
/////////////////////////////////////////////////////////////////////////////////
// query for the edited records - assert that id 1 & 2 were updated, 3 was not //
/////////////////////////////////////////////////////////////////////////////////
List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
assertEquals("Johnny", records.get(0).getValueString("firstName"));
assertEquals("Johnny", records.get(1).getValueString("firstName"));
assertEquals("James", records.get(2).getValueString("firstName"));
assertEquals("1909-01-09", records.get(0).getValueString("birthDate"));
assertEquals("1909-01-09", records.get(1).getValueString("birthDate"));
assertNull(records.get(2).getValueString("birthDate"));
assertNull(records.get(0).getValue("email"));
assertNull(records.get(1).getValue("email"));
assertEquals("james.maes@kingsrook.com", records.get(2).getValue("email"));
}
}

114
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java
View File

@ -1,114 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/*******************************************************************************
** Unit test for BulkInsertReceiveFileStep
*******************************************************************************/
class BulkInsertReceiveFileStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
////////////////////////////////////////////////////////////////
// create an uploaded file, similar to how an http server may //
////////////////////////////////////////////////////////////////
QUploadedFile qUploadedFile = new QUploadedFile();
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
qUploadedFile.setFilename("test.csv");
UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
TempFileStateProvider.getInstance().put(key, qUploadedFile);
////////////////////////////
// setup and run the step //
////////////////////////////
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkInsertReceiveFileStep().run(stepInput, stepOutput);
List<QRecord> records = stepOutput.getRecords();
assertEquals(2, records.size());
assertEquals("John", records.get(0).getValueString("firstName"));
assertEquals("Jane", records.get(1).getValueString("firstName"));
assertNull(records.get(0).getValue("id"));
assertNull(records.get(1).getValue("id"));
assertEquals(2, stepOutput.getValueInteger("noOfFileRows"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBadFileType() throws QException
{
////////////////////////////////////////////////////////////////
// create an uploaded file, similar to how an http server may //
////////////////////////////////////////////////////////////////
QUploadedFile qUploadedFile = new QUploadedFile();
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); // todo - this is NOT excel content...
qUploadedFile.setFilename("test.xslx");
UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
TempFileStateProvider.getInstance().put(key, qUploadedFile);
////////////////////////////
// setup and run the step //
////////////////////////////
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
assertThrows(QUserFacingException.class, () ->
{
new BulkInsertReceiveFileStep().run(stepInput, stepOutput);
});
}
}

80
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java
View File

@ -1,80 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for BulkInsertStoreRecordsStep
*******************************************************************************/
class BulkInsertStoreRecordsStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
////////////////////////////////////////////////////////////////
// create an uploaded file, similar to how an http server may //
////////////////////////////////////////////////////////////////
QUploadedFile qUploadedFile = new QUploadedFile();
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
qUploadedFile.setFilename("test.csv");
UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
TempFileStateProvider.getInstance().put(key, qUploadedFile);
////////////////////////////
// setup and run the step //
////////////////////////////
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkInsertStoreRecordsStep().run(stepInput, stepOutput);
List<QRecord> records = stepOutput.getRecords();
assertEquals(2, records.size());
assertEquals("John", records.get(0).getValueString("firstName"));
assertEquals("Jane", records.get(1).getValueString("firstName"));
assertNotNull(records.get(0).getValue("id"));
assertNotNull(records.get(1).getValue("id"));
}
}

121
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java Normal file
View File

@ -0,0 +1,121 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for full bulk insert process
*******************************************************************************/
class BulkInsertTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
///////////////////////////////////////
// make sure table is empty to start //
///////////////////////////////////////
assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
////////////////////////////////////////////////////////////////
// create an uploaded file, similar to how an http server may //
////////////////////////////////////////////////////////////////
QUploadedFile qUploadedFile = new QUploadedFile();
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
qUploadedFile.setFilename("test.csv");
UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile);
RunProcessInput runProcessInput = new RunProcessInput(TestUtils.defineInstance());
runProcessInput.setSession(TestUtils.getMockSession());
runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert");
RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
String processUUID = runProcessOutput.getProcessUUID();
runProcessInput.setProcessUUID(processUUID);
runProcessInput.setStartAfterStep("upload");
runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, uploadedFileKey);
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
assertThat(runProcessOutput.getException()).isEmpty();
////////////////////////////////////
// query for the inserted records //
////////////////////////////////////
List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
assertEquals("John", records.get(0).getValueString("firstName"));
assertEquals("Jane", records.get(1).getValueString("firstName"));
assertNotNull(records.get(0).getValue("id"));
assertNotNull(records.get(1).getValue("id"));
}
}

16
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -29,7 +29,6 @@ import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarChart;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge;
@ -164,7 +163,6 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
@ -207,6 +205,20 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static void insertRecords(QInstance qInstance, QTableMetaData table, List<QRecord> records) throws QException
{
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(table.getName());
insertInput.setRecords(records);
new InsertAction().execute(insertInput);
}
/*******************************************************************************
**
*******************************************************************************/