diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 5ac811a9..3af8b35d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -74,6 +74,10 @@ public class QValueFormatter { return formatValue(field, ValueUtils.getValueAsBigDecimal(value)); } + else if(e.getMessage().equals("f != java.lang.String")) + { + return formatValue(field, ValueUtils.getValueAsBigDecimal(value)); + } else if(e.getMessage().equals("d != java.math.BigDecimal")) { return formatValue(field, ValueUtils.getValueAsInteger(value)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java index 98f85478..77b1e85d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -62,8 +63,7 @@ public class CsvToQRecordAdapter *******************************************************************************/ public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping mapping, Consumer recordCustomizer) { - this.recordPipe = recordPipe; - doBuildRecordsFromCsv(csv, table, mapping, recordCustomizer); + buildRecordsFromCsv(new InputWrapper().withRecordPipe(recordPipe).withCsv(csv).withTable(table).withMapping(mapping).withRecordCustomizer(recordCustomizer)); } @@ -75,8 +75,7 @@ public class CsvToQRecordAdapter *******************************************************************************/ public List buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping) { - this.recordList = new ArrayList<>(); - doBuildRecordsFromCsv(csv, table, mapping, null); + buildRecordsFromCsv(new InputWrapper().withCsv(csv).withTable(table).withMapping(mapping)); return (recordList); } @@ -88,13 +87,29 @@ public class CsvToQRecordAdapter ** ** todo - meta-data validation, type handling *******************************************************************************/ - public void doBuildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping, Consumer recordCustomizer) + public void buildRecordsFromCsv(InputWrapper inputWrapper) { + String csv = inputWrapper.getCsv(); + AbstractQFieldMapping mapping = inputWrapper.getMapping(); + Consumer recordCustomizer = inputWrapper.getRecordCustomizer(); + QTableMetaData table = inputWrapper.getTable(); + Integer limit = inputWrapper.getLimit(); + if(!StringUtils.hasContent(csv)) { throw (new IllegalArgumentException("Empty csv value was provided.")); } + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // if caller supplied a record pipe, use it -- but if it's null, then create a recordList to populate. // + // see addRecord method for usage. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + this.recordPipe = inputWrapper.getRecordPipe(); + if(this.recordPipe == null) + { + this.recordList = new ArrayList<>(); + } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // once, from a DOS csv file (that had come from Excel), we had a "" character (FEFF, Byte-order marker) at the start of a // // CSV, which caused our first header to not match... So, let us strip away any FEFF or FFFE's at the start of CSV strings. // @@ -120,9 +135,12 @@ public class CsvToQRecordAdapter List headers = csvParser.getHeaderNames(); headers = makeHeadersUnique(headers); - List csvRecords = csvParser.getRecords(); - for(CSVRecord csvRecord : csvRecords) + Iterator csvIterator = csvParser.iterator(); + int recordCount = 0; + while(csvIterator.hasNext()) { + CSVRecord csvRecord = csvIterator.next(); + ////////////////////////////////////////////////////////////////// // put values from the CSV record into a map of header -> value // ////////////////////////////////////////////////////////////////// @@ -144,6 +162,12 @@ public class CsvToQRecordAdapter runRecordCustomizer(recordCustomizer, qRecord); addRecord(qRecord); + + recordCount++; + if(limit != null && recordCount > limit) + { + break; + } } } else if(AbstractQFieldMapping.SourceType.INDEX.equals(mapping.getSourceType())) @@ -155,9 +179,12 @@ public class CsvToQRecordAdapter CSVFormat.DEFAULT .withTrim()); - List csvRecords = csvParser.getRecords(); - for(CSVRecord csvRecord : csvRecords) + Iterator csvIterator = csvParser.iterator(); + int recordCount = 0; + while(csvIterator.hasNext()) { + CSVRecord csvRecord = csvIterator.next(); + ///////////////////////////////////////////////////////////////// // put values from the CSV record into a map of index -> value // ///////////////////////////////////////////////////////////////// @@ -180,6 +207,12 @@ public class CsvToQRecordAdapter runRecordCustomizer(recordCustomizer, qRecord); addRecord(qRecord); + + recordCount++; + if(limit != null && recordCount > limit) + { + break; + } } } else @@ -261,4 +294,241 @@ public class CsvToQRecordAdapter } } + + + /******************************************************************************* + ** Getter for recordList - note - only is valid if you don't supply a pipe in + ** the input. If you do supply a pipe, then you get an exception if you call here! + ** + *******************************************************************************/ + public List getRecordList() + { + if(recordPipe != null) + { + throw (new IllegalStateException("getRecordList called on a CSVToQRecordAdapter that ran with a recordPipe.")); + } + + return recordList; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class InputWrapper + { + private RecordPipe recordPipe; + private String csv; + private QTableMetaData table; + private AbstractQFieldMapping mapping; + private Consumer recordCustomizer; + private Integer limit; + + + + /******************************************************************************* + ** Getter for recordPipe + ** + *******************************************************************************/ + public RecordPipe getRecordPipe() + { + return recordPipe; + } + + + + /******************************************************************************* + ** Setter for recordPipe + ** + *******************************************************************************/ + public void setRecordPipe(RecordPipe recordPipe) + { + this.recordPipe = recordPipe; + } + + + + /******************************************************************************* + ** Fluent setter for recordPipe + ** + *******************************************************************************/ + public InputWrapper withRecordPipe(RecordPipe recordPipe) + { + this.recordPipe = recordPipe; + return (this); + } + + + + /******************************************************************************* + ** Getter for csv + ** + *******************************************************************************/ + public String getCsv() + { + return csv; + } + + + + /******************************************************************************* + ** Setter for csv + ** + *******************************************************************************/ + public void setCsv(String csv) + { + this.csv = csv; + } + + + + /******************************************************************************* + ** Fluent setter for csv + ** + *******************************************************************************/ + public InputWrapper withCsv(String csv) + { + this.csv = csv; + return (this); + } + + + + /******************************************************************************* + ** Getter for table + ** + *******************************************************************************/ + public QTableMetaData getTable() + { + return table; + } + + + + /******************************************************************************* + ** Setter for table + ** + *******************************************************************************/ + public void setTable(QTableMetaData table) + { + this.table = table; + } + + + + /******************************************************************************* + ** Fluent setter for table + ** + *******************************************************************************/ + public InputWrapper withTable(QTableMetaData table) + { + this.table = table; + return (this); + } + + + + /******************************************************************************* + ** Getter for mapping + ** + *******************************************************************************/ + public AbstractQFieldMapping getMapping() + { + return mapping; + } + + + + /******************************************************************************* + ** Setter for mapping + ** + *******************************************************************************/ + public void setMapping(AbstractQFieldMapping mapping) + { + this.mapping = mapping; + } + + + + /******************************************************************************* + ** Fluent setter for mapping + ** + *******************************************************************************/ + public InputWrapper withMapping(AbstractQFieldMapping mapping) + { + this.mapping = mapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for recordCustomizer + ** + *******************************************************************************/ + public Consumer getRecordCustomizer() + { + return recordCustomizer; + } + + + + /******************************************************************************* + ** Setter for recordCustomizer + ** + *******************************************************************************/ + public void setRecordCustomizer(Consumer recordCustomizer) + { + this.recordCustomizer = recordCustomizer; + } + + + + /******************************************************************************* + ** Fluent setter for recordCustomizer + ** + *******************************************************************************/ + public InputWrapper withRecordCustomizer(Consumer recordCustomizer) + { + this.recordCustomizer = recordCustomizer; + return (this); + } + + + + /******************************************************************************* + ** Getter for limit + ** + *******************************************************************************/ + public Integer getLimit() + { + return limit; + } + + + + /******************************************************************************* + ** Setter for limit + ** + *******************************************************************************/ + public void setLimit(Integer limit) + { + this.limit = limit; + } + + + + /******************************************************************************* + ** Fluent setter for limit + ** + *******************************************************************************/ + public InputWrapper withLimit(Integer limit) + { + this.limit = limit; + return (this); + } + + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 765d26b8..d56f830a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -22,38 +22,39 @@ package com.kingsrook.qqq.backend.core.instances; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteStoreStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditReceiveValuesStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditStoreRecordsStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertStoreRecordsStep; -import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.logging.log4j.LogManager; @@ -293,6 +294,20 @@ public class QInstanceEnricher *******************************************************************************/ private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + BulkInsertExtractStep.class, + BulkInsertTransformStep.class, + LoadViaInsertStep.class, + values + ) + .withName(processName) + .withLabel(table.getLabel() + " Bulk Insert") + .withTableName(table.getName()) + .withIsHidden(true); + List editableFields = table.getFields().values().stream() .filter(QFieldMetaData::getIsEditable) .toList(); @@ -307,54 +322,13 @@ public class QInstanceEnricher .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true)) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) - .withValue("text", "Upload a CSV file with the following columns: " + fieldsForHelpText)) + .withValue("previewText", "file upload instructions") + .withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText)) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); - QBackendStepMetaData receiveFileStep = new QBackendStepMetaData() - .withName("receiveFile") - .withCode(new QCodeReference(BulkInsertReceiveFileStep.class)) - .withOutputMetaData(new QFunctionOutputMetaData() - .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER)))); - - QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData() - .withName("review") - .withRecordListFields(editableFields) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below were parsed from your file, and will be inserted if you click Submit.")) - .withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); - - QBackendStepMetaData storeStep = new QBackendStepMetaData() - .withName("storeRecords") - .withCode(new QCodeReference(BulkInsertStoreRecordsStep.class)) - .withOutputMetaData(new QFunctionOutputMetaData() - .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER)))); - - QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData() - .withName("results") - .withRecordListFields(new ArrayList<>(table.getFields().values())) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below have been inserted.")) - .withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); - - qInstance.addProcess( - new QProcessMetaData() - .withName(processName) - .withLabel(table.getLabel() + " Bulk Insert") - .withTableName(table.getName()) - .withIsHidden(true) - .withStepList(List.of( - uploadScreen, - receiveFileStep, - reviewScreen, - storeStep, - resultsScreen - ))); + process.addStep(0, uploadScreen); + process.getFrontendStep("review").setRecordListFields(editableFields); + qInstance.addProcess(process); } @@ -364,6 +338,22 @@ public class QInstanceEnricher *******************************************************************************/ private void defineTableBulkEdit(QInstance qInstance, QTableMetaData table, String processName) { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + ExtractViaQueryStep.class, + BulkEditTransformStep.class, + LoadViaUpdateStep.class, + values + ) + .withName(processName) + .withLabel(table.getLabel() + " Bulk Edit") + .withTableName(table.getName()) + .withIsHidden(true); + List editableFields = table.getFields().values().stream() .filter(QFieldMetaData::getIsEditable) .toList(); @@ -381,54 +371,9 @@ public class QInstanceEnricher Fields whose switches are off will not be updated.""")) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_EDIT_FORM)); - QBackendStepMetaData receiveValuesStep = new QBackendStepMetaData() - .withName("receiveValues") - .withCode(new QCodeReference(BulkEditReceiveValuesStep.class)) - .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName(table.getName())) - .withField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, QFieldType.STRING)) - .withFields(editableFields)); - - QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData() - .withName("review") - .withRecordListFields(editableFields) - .withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below will be updated if you click Submit.")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); - - QBackendStepMetaData storeStep = new QBackendStepMetaData() - .withName("storeRecords") - .withCode(new QCodeReference(BulkEditStoreRecordsStep.class)) - .withOutputMetaData(new QFunctionOutputMetaData() - .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER)))); - - QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData() - .withName("results") - .withRecordListFields(new ArrayList<>(table.getFields().values())) - .withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below have been updated.")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); - - qInstance.addProcess( - new QProcessMetaData() - .withName(processName) - .withLabel(table.getLabel() + " Bulk Edit") - .withTableName(table.getName()) - .withIsHidden(true) - .withStepList(List.of( - LoadInitialRecordsStep.defineMetaData(table.getName()), - editScreen, - receiveValuesStep, - reviewScreen, - storeStep, - resultsScreen - ))); + process.addStep(0, editScreen); + process.getFrontendStep("review").setRecordListFields(editableFields); + qInstance.addProcess(process); } @@ -438,38 +383,26 @@ public class QInstanceEnricher *******************************************************************************/ private void defineTableBulkDelete(QInstance qInstance, QTableMetaData table, String processName) { - QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData() - .withName("review") - .withRecordListFields(new ArrayList<>(table.getFields().values())) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below will be deleted if you click Submit.")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_DELETE); - QBackendStepMetaData storeStep = new QBackendStepMetaData() - .withName("delete") - .withCode(new QCodeReference(BulkDeleteStoreStep.class)); + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + ExtractViaQueryStep.class, + BulkDeleteTransformStep.class, + LoadViaDeleteStep.class, + values + ) + .withName(processName) + .withLabel(table.getLabel() + " Bulk Delete") + .withTableName(table.getName()) + .withIsHidden(true); - QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData() - .withName("results") - .withRecordListFields(new ArrayList<>(table.getFields().values())) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below have been deleted.")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); + List tableFields = table.getFields().values().stream().toList(); + process.getFrontendStep("review").setRecordListFields(tableFields); - qInstance.addProcess( - new QProcessMetaData() - .withName(processName) - .withLabel(table.getLabel() + " Bulk Delete") - .withTableName(table.getName()) - .withIsHidden(true) - .withStepList(List.of( - LoadInitialRecordsStep.defineMetaData(table.getName()), - reviewScreen, - storeStep, - resultsScreen - ))); + qInstance.addProcess(process); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java index 7a21349f..12aa596b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -35,7 +35,7 @@ import java.util.List; public class ProcessSummaryLine implements Serializable { private Status status; - private Integer count; + private Integer count = 0; private String message; ////////////////////////////////////////////////////////////////////////// @@ -77,7 +77,16 @@ public class ProcessSummaryLine implements Serializable { this.status = status; this.message = message; - this.count = 0; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLine(Status status) + { + this.status = status; } @@ -185,12 +194,22 @@ public class ProcessSummaryLine implements Serializable ** *******************************************************************************/ public void incrementCount() + { + incrementCount(1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void incrementCount(int amount) { if(count == null) { count = 0; } - count++; + count += amount; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java index 40da4720..ec7c1805 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.delete; import java.io.Serializable; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -35,8 +36,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; *******************************************************************************/ public class DeleteInput extends AbstractTableActionInput { - private List primaryKeys; - private QQueryFilter queryFilter; + private QBackendTransaction transaction; + private List primaryKeys; + private QQueryFilter queryFilter; @@ -59,6 +61,40 @@ public class DeleteInput extends AbstractTableActionInput + /******************************************************************************* + ** Getter for transaction + ** + *******************************************************************************/ + public QBackendTransaction getTransaction() + { + return transaction; + } + + + + /******************************************************************************* + ** Setter for transaction + ** + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + + + + /******************************************************************************* + ** Fluent setter for transaction + ** + *******************************************************************************/ + public DeleteInput withTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + return (this); + } + + + /******************************************************************************* ** Getter for ids ** @@ -92,6 +128,7 @@ public class DeleteInput extends AbstractTableActionInput } + /******************************************************************************* ** Getter for queryFilter ** @@ -113,6 +150,7 @@ public class DeleteInput extends AbstractTableActionInput } + /******************************************************************************* ** Fluent setter for queryFilter ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java deleted file mode 100644 index c8cf5f11..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; - - -import java.io.IOException; -import java.io.Serializable; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; - - -/******************************************************************************* - ** Backend step to do a bulk delete. - *******************************************************************************/ -public class BulkDeleteStoreStep implements BackendStep -{ - public static final String ERROR_COUNT = "errorCount"; - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records..."); - runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal(); - - DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance()); - deleteInput.setSession(runBackendStepInput.getSession()); - deleteInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); - deleteInput.setTableName(runBackendStepInput.getTableName()); - - String queryFilterJSON = runBackendStepInput.getValueString("queryFilterJSON"); - if(StringUtils.hasContent(queryFilterJSON)) - { - try - { - deleteInput.setQueryFilter(JsonUtils.toObject(queryFilterJSON, QQueryFilter.class)); - } - catch(IOException e) - { - throw (new QException("Error loading record query filter from process", e)); - } - } - else if(CollectionUtils.nullSafeHasContents(runBackendStepInput.getRecords())) - { - String primaryKeyField = runBackendStepInput.getTable().getPrimaryKeyField(); - List primaryKeyList = runBackendStepInput.getRecords().stream() - .map(r -> r.getValue(primaryKeyField)) - .toList(); - deleteInput.setPrimaryKeys(primaryKeyList); - } - else - { - throw (new QException("Missing required inputs (queryFilterJSON or record list)")); - } - - DeleteAction deleteAction = new DeleteAction(); - DeleteOutput deleteOutput = deleteAction.execute(deleteInput); - - List recordsWithErrors = Objects.requireNonNullElse(deleteOutput.getRecordsWithErrors(), Collections.emptyList()); - runBackendStepOutput.addValue(ERROR_COUNT, recordsWithErrors.size()); - - runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java new file mode 100644 index 00000000..64c1205a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java @@ -0,0 +1,126 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + + +/******************************************************************************* + ** Transform step for generic table bulk-insert ETL process + *******************************************************************************/ +public class BulkDeleteTransformStep extends AbstractTransformStep +{ + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + + private String tableLabel; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + /////////////////////////////////////////////////////// + // capture the table label - for the process summary // + /////////////////////////////////////////////////////// + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + if(table != null) + { + tableLabel = table.getLabel(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // + // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) + { + if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record " + "%,d".formatted(okSummary.getCount())); + } + else + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record"); + } + } + else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE)) + { + if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " record " + "%,d".formatted(okSummary.getCount())); + } + else + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " records"); + } + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // no transformation needs done - just pass records through from input to output, and assume all are OK // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); + okSummary.incrementCount(runBackendStepInput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage(tableLabel + " records were deleted."); + } + else + { + okSummary.setMessage(tableLabel + " records will be deleted."); + } + + ArrayList rs = new ArrayList<>(); + rs.add(okSummary); + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java deleted file mode 100644 index 8253999a..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.io.Serializable; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; - - -/******************************************************************************* - ** Backend step to receive values for a bulk edit. - *******************************************************************************/ -public class BulkEditReceiveValuesStep implements BackendStep -{ - public static final String FIELD_ENABLED_FIELDS = "bulkEditEnabledFields"; - public static final String FIELD_VALUES_BEING_UPDATED = "valuesBeingUpdated"; - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - String enabledFieldsString = runBackendStepInput.getValueString(FIELD_ENABLED_FIELDS); - String[] enabledFields = enabledFieldsString.split(","); - - //////////////////////////////////////////////////////////////////////////////////////////// - // put the value in all the records (note, this is just for display on the review screen, // - // and/or if we wanted to do some validation - this is NOT what will be store, as the // - // Update action only wants fields that are being changed. // - //////////////////////////////////////////////////////////////////////////////////////////// - for(QRecord record : runBackendStepInput.getRecords()) - { - for(String fieldName : enabledFields) - { - Serializable value = runBackendStepInput.getValue(fieldName); - record.setValue(fieldName, value); - } - } - - BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "will be"); - - runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); - } -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java deleted file mode 100644 index f52a7335..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; - - -/******************************************************************************* - ** Backend step to store the records from a bulk insert file - *******************************************************************************/ -public class BulkEditStoreRecordsStep implements BackendStep -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - String enabledFieldsString = runBackendStepInput.getValueString(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS); - String[] enabledFields = enabledFieldsString.split(","); - QTableMetaData table = runBackendStepInput.getTable(); - List recordsToUpdate = new ArrayList<>(); - - runBackendStepInput.getAsyncJobCallback().updateStatus("Updating values in records..."); - int i = 1; - for(QRecord record : runBackendStepInput.getRecords()) - { - runBackendStepInput.getAsyncJobCallback().updateStatus(i++, runBackendStepInput.getRecords().size()); - QRecord recordToUpdate = new QRecord(); - recordsToUpdate.add(recordToUpdate); - - recordToUpdate.setValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField())); - for(String fieldName : enabledFields) - { - Serializable value = runBackendStepInput.getValue(fieldName); - recordToUpdate.setValue(fieldName, value); - } - } - - runBackendStepInput.getAsyncJobCallback().updateStatus("Storing updated records..."); - runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal(); - - UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); - updateInput.setSession(runBackendStepInput.getSession()); - updateInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); - updateInput.setAreAllValuesBeingUpdatedTheSame(true); - updateInput.setTableName(runBackendStepInput.getTableName()); - updateInput.setRecords(recordsToUpdate); - - UpdateAction updateAction = new UpdateAction(); - UpdateOutput updateOutput = updateAction.execute(updateInput); - - runBackendStepOutput.setRecords(updateOutput.getRecords()); - - BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "was"); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java new file mode 100644 index 00000000..954b562c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java @@ -0,0 +1,215 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Transform step for generic table bulk-edit ETL process + *******************************************************************************/ +public class BulkEditTransformStep extends AbstractTransformStep +{ + public static final String FIELD_ENABLED_FIELDS = "bulkEditEnabledFields"; + + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + private List infoSummaries = new ArrayList<>(); + + private QTableMetaData table; + private String tableLabel; + private String[] enabledFields; + + private boolean isValidateStep; + private boolean isExecuteStep; + private boolean haveRecordCount; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + /////////////////////////////////////////////////////// + // capture the table label - for the process summary // + /////////////////////////////////////////////////////// + table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + if(table != null) + { + tableLabel = table.getLabel(); + } + + String enabledFieldsString = runBackendStepInput.getValueString(FIELD_ENABLED_FIELDS); + enabledFields = enabledFieldsString.split(","); + + isValidateStep = runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + isExecuteStep = runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE); + haveRecordCount = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) != null; + + buildInfoSummaryLines(runBackendStepInput, enabledFields); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // + // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(isValidateStep) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records"); + if(!haveRecordCount) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing record " + "%,d".formatted(okSummary.getCount())); + } + } + else if(isExecuteStep) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Editing " + tableLabel + " records"); + if(!haveRecordCount) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Editing " + tableLabel + " record " + "%,d".formatted(okSummary.getCount())); + } + } + + List outputRecords = new ArrayList<>(); + if(isExecuteStep) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for the execute step - create new record objects, just with the primary key, and the fields being updated. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : runBackendStepInput.getRecords()) + { + QRecord recordToUpdate = new QRecord(); + recordToUpdate.setValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField())); + outputRecords.add(recordToUpdate); + setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, recordToUpdate); + } + } + else + { + //////////////////////////////////////////////////////////////////////////////////////////// + // put the value in all the records (note, this is just for display on the review screen, // + // and/or if we wanted to do some validation - this is NOT what will be store, as the // + // Update action only wants fields that are being changed. // + //////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : runBackendStepInput.getRecords()) + { + outputRecords.add(record); + setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, record); + } + } + runBackendStepOutput.setRecords(outputRecords); + okSummary.incrementCount(runBackendStepInput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void buildInfoSummaryLines(RunBackendStepInput runBackendStepInput, String[] enabledFields) + { + QValueFormatter qValueFormatter = new QValueFormatter(); + for(String fieldName : enabledFields) + { + QFieldMetaData field = table.getField(fieldName); + String label = field.getLabel(); + Serializable value = runBackendStepInput.getValue(fieldName); + + ProcessSummaryLine summaryLine = new ProcessSummaryLine(Status.INFO); + summaryLine.setCount(null); + infoSummaries.add(summaryLine); + + String verb = isExecuteStep ? "was" : "will be"; + if(StringUtils.hasContent(ValueUtils.getValueAsString(value))) + { + String formattedValue = qValueFormatter.formatValue(field, value); // todo - PVS! + summaryLine.setMessage(label + " " + verb + " set to: " + formattedValue); + } + else + { + summaryLine.setMessage(label + " " + verb + " cleared out"); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setUpdatedFieldsInRecord(RunBackendStepInput runBackendStepInput, String[] enabledFields, QRecord record) + { + for(String fieldName : enabledFields) + { + Serializable value = runBackendStepInput.getValue(fieldName); + record.setValue(fieldName, value); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage(tableLabel + " records were edited."); + } + else + { + okSummary.setMessage(tableLabel + " records will be edited."); + } + + ArrayList rs = new ArrayList<>(); + rs.add(okSummary); + rs.addAll(infoSummaries); + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java deleted file mode 100644 index 71eeb69f..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; - - -/******************************************************************************* - ** Utility methods used for Bulk Edit steps - *******************************************************************************/ -public class BulkEditUtils -{ - - /******************************************************************************* - ** - *******************************************************************************/ - public static void setFieldValuesBeingUpdated(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, String[] enabledFields, String verb) - { - ///////////////////////////////////////////////////////////////////// - // build the string to show the user what fields are being changed // - ///////////////////////////////////////////////////////////////////// - List valuesBeingUpdated = new ArrayList<>(); - QTableMetaData table = runBackendStepInput.getTable(); - for(String fieldName : enabledFields) - { - String label = table.getField(fieldName).getLabel(); - Serializable value = runBackendStepInput.getValue(fieldName); - - if(StringUtils.hasContent(ValueUtils.getValueAsString(value))) - { - valuesBeingUpdated.add(label + " " + verb + " set to: " + value); - } - else - { - valuesBeingUpdated.add(label + " " + verb + " cleared out"); - } - } - runBackendStepOutput.addValue(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated)); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java similarity index 70% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index 4d4182bb..d8850d40 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -31,23 +31,22 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; import com.kingsrook.qqq.backend.core.state.AbstractStateKey; import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; /******************************************************************************* - ** Utility methods used by bulk insert steps + ** Extract step for generic table bulk-insert ETL process *******************************************************************************/ -public class BulkInsertUtils +public class BulkInsertExtractStep extends AbstractExtractStep { - /******************************************************************************* - ** - *******************************************************************************/ - static List getQRecordsFromFile(RunBackendStepInput runBackendStepInput) throws QException + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME); Optional optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey); @@ -70,33 +69,35 @@ public class BulkInsertUtils mapping.addMapping(entry.getKey(), entry.getValue().getLabel()); } - List qRecords; + ////////////////////////////////////////////////////////////////////////// + // get the non-editable fields - they'll be blanked out in a customizer // + ////////////////////////////////////////////////////////////////////////// + List nonEditableFields = table.getFields().values().stream() + .filter(f -> !f.getIsEditable()) + .toList(); + if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) { - qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping); + new CsvToQRecordAdapter().buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() + .withRecordPipe(getRecordPipe()) + .withLimit(getLimit()) + .withCsv(new String(bytes)) + .withTable(runBackendStepInput.getInstance().getTable(tableName)) + .withMapping(mapping) + .withRecordCustomizer((record) -> + { + //////////////////////////////////////////// + // remove values from non-editable fields // + //////////////////////////////////////////// + for(QFieldMetaData nonEditableField : nonEditableFields) + { + record.setValue(nonEditableField.getName(), null); + } + })); } else { throw (new QUserFacingException("Unsupported file type.")); } - - //////////////////////////////////////////////// - // remove values from any non-editable fields // - //////////////////////////////////////////////// - List nonEditableFields = table.getFields().values().stream() - .filter(f -> !f.getIsEditable()) - .toList(); - if(!nonEditableFields.isEmpty()) - { - for(QRecord qRecord : qRecords) - { - for(QFieldMetaData nonEditableField : nonEditableFields) - { - qRecord.setValue(nonEditableField.getName(), null); - } - } - } - - return (qRecords); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java deleted file mode 100644 index 7844ef9c..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; - - -/******************************************************************************* - ** Backend step to receive a bulk-insert upload file - *******************************************************************************/ -public class BulkInsertReceiveFileStep implements BackendStep -{ - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - List qRecords = BulkInsertUtils.getQRecordsFromFile(runBackendStepInput); - - runBackendStepOutput.addValue("noOfFileRows", qRecords.size()); - runBackendStepOutput.setRecords(qRecords); - } -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java new file mode 100644 index 00000000..3c0fabf3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + + +/******************************************************************************* + ** Transform step for generic table bulk-insert ETL process + *******************************************************************************/ +public class BulkInsertTransformStep extends AbstractTransformStep +{ + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + + private String tableLabel; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + /////////////////////////////////////////////////////// + // capture the table label - for the process summary // + /////////////////////////////////////////////////////// + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + if(table != null) + { + tableLabel = table.getLabel(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // + // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(okSummary.getCount())); + } + else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE)) + { + if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " record " + "%,d".formatted(okSummary.getCount())); + } + else + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " records"); + } + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // no transformation needs done - just pass records through from input to output, and assume all are OK // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); + okSummary.incrementCount(runBackendStepInput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage(tableLabel + " records were inserted."); + } + else + { + okSummary.setMessage(tableLabel + " records will be inserted."); + } + + ArrayList rs = new ArrayList<>(); + rs.add(okSummary); + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index 888857d8..a7a86049 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.IOException; import java.io.Serializable; +import java.util.Arrays; +import java.util.List; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -31,9 +33,13 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -94,10 +100,12 @@ public class ExtractViaQueryStep extends AbstractExtractStep *******************************************************************************/ protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException { + String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson"); + Serializable defaultQueryFilter = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER); + ////////////////////////////////////////////////////////////////////////////////////// // if the queryFilterJson field is populated, read the filter from it and return it // ////////////////////////////////////////////////////////////////////////////////////// - String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson"); if(queryFilterJson != null) { return getQueryFilterFromJson(queryFilterJson, "Error loading query filter from json field"); @@ -111,20 +119,41 @@ public class ExtractViaQueryStep extends AbstractExtractStep runBackendStepInput.addValue("queryFilterJson", JsonUtils.toJson(queryFilter)); return (queryFilter); } - else + else if(defaultQueryFilter instanceof QQueryFilter filter) { - ///////////////////////////////////////////////////// - // else, see if a defaultQueryFilter was specified // - ///////////////////////////////////////////////////// - Serializable defaultQueryFilter = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER); - if(defaultQueryFilter instanceof QQueryFilter filter) + ///////////////////////////////////////////////////////////////////////////// + // else, see if a defaultQueryFilter was specified as a QueryFilter object // + ///////////////////////////////////////////////////////////////////////////// + return (filter); + } + else if(defaultQueryFilter instanceof String string) + { + ///////////////////////////////////////////////////////////////////////////// + // else, see if a defaultQueryFilter was specified as a JSON string + ///////////////////////////////////////////////////////////////////////////// + return getQueryFilterFromJson(string, "Error loading default query filter from json"); + } + else if(StringUtils.hasContent(runBackendStepInput.getValueString("filterJSON"))) + { + /////////////////////////////////////////////////////////////////////// + // else, check for filterJSON from a frontend launching of a process // + /////////////////////////////////////////////////////////////////////// + return getQueryFilterFromJson(runBackendStepInput.getValueString("filterJSON"), "Error loading default query filter from json"); + } + else if(StringUtils.hasContent(runBackendStepInput.getValueString("recordIds"))) + { + ////////////////////////////////////////////////////////////////////// + // else, check for recordIds from a frontend launching of a process // + ////////////////////////////////////////////////////////////////////// + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); + if(table == null) { - return (filter); - } - else if(defaultQueryFilter instanceof String string) - { - return getQueryFilterFromJson(string, "Error loading default query filter from json"); + throw (new QException("source table name was not set - could not load records by id")); } + String recordIds = runBackendStepInput.getValueString("recordIds"); + Serializable[] split = recordIds.split(","); + List idStrings = Arrays.stream(split).toList(); + return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, idStrings))); } throw (new QException("Could not find query filter for Extract step.")); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java similarity index 51% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStep.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java index 1f6710d5..34aa0d3b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java @@ -19,41 +19,62 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import java.util.Optional; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; /******************************************************************************* - ** Backend step to store the records from a bulk insert file + ** Generic implementation of a LoadStep - that runs a Delete action for a + ** specified table. *******************************************************************************/ -public class BulkInsertStoreRecordsStep implements BackendStep +public class LoadViaDeleteStep extends AbstractLoadStep { + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. ** *******************************************************************************/ @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - List qRecords = BulkInsertUtils.getQRecordsFromFile(runBackendStepInput); + QTableMetaData table = runBackendStepInput.getTable(); + DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance()); + deleteInput.setSession(runBackendStepInput.getSession()); + deleteInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); + deleteInput.setPrimaryKeys(runBackendStepInput.getRecords().stream().map(r -> r.getValue(table.getPrimaryKeyField())).collect(Collectors.toList())); + // todo? can make more efficient deletes, maybe? deleteInput.setQueryFilter(); + getTransaction().ifPresent(deleteInput::setTransaction); + new DeleteAction().execute(deleteInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Optional openTransaction(RunBackendStepInput runBackendStepInput) throws QException + { InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); insertInput.setSession(runBackendStepInput.getSession()); - insertInput.setTableName(runBackendStepInput.getTableName()); - insertInput.setRecords(qRecords); + insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); - InsertAction insertAction = new InsertAction(); - InsertOutput insertOutput = insertAction.execute(insertInput); - - runBackendStepOutput.setRecords(insertOutput.getRecords()); + return (Optional.of(new InsertAction().openTransaction(insertInput))); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 5d872b7c..9b982921 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -82,6 +82,7 @@ public class StreamedETLWithFrontendProcess public static final String DEFAULT_PREVIEW_MESSAGE_FOR_INSERT = "This is a preview of the records that will be created."; public static final String DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE = "This is a preview of the records that will be updated."; + public static final String DEFAULT_PREVIEW_MESSAGE_FOR_DELETE = "This is a preview of the records that will be deleted."; public static final String FIELD_PREVIEW_MESSAGE = "previewMessage"; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java deleted file mode 100644 index 483e2edf..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; - - -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - - -/******************************************************************************* - ** Unit test for BulkDeleteStoreStep - *******************************************************************************/ -class BulkDeleteStoreStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testWithoutFilter() throws QException - { - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.setRecords(TestUtils.queryTable(TestUtils.defineTablePerson().getName())); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkDeleteStoreStep().run(stepInput, stepOutput); - assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT)); - } - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testWithFilter() throws QException - { - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue("queryFilterJSON", JsonUtils.toJson(new QQueryFilter())); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkDeleteStoreStep().run(stepInput, stepOutput); - assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT)); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTest.java new file mode 100644 index 00000000..1ede46da --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTest.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for full bulk edit process + *******************************************************************************/ +class BulkDeleteTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ////////////////////////////// + // insert some test records // + ////////////////////////////// + QInstance qInstance = TestUtils.defineInstance(); + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("email", "darin.kelkhoff@kingsrook.com"), + new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("email", "tim.chamberlain@kingsrook.com"), + new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("lastName", "Maes").withValue("email", "james.maes@kingsrook.com") + )); + + ////////////////////////////////// + // set up the run-process input // + ////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(qInstance); + runProcessInput.setSession(TestUtils.getMockSession()); + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkDelete"); + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, + new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 3)))); + + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + runProcessInput.addValue("processUUID", processUUID); + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class); + + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + @SuppressWarnings("unchecked") + List processSummaryLines = (List) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + assertThat(processSummaryLines).hasSize(1); + assertThat(processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.OK))).hasSize(1); + + ////////////////////////////////////////////////////////////// + // query for the records - assert that only id=2 exists now // + ////////////////////////////////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals(2, records.get(0).getValueInteger("id")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStepTest.java deleted file mode 100644 index 0619d13a..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStepTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - - -/******************************************************************************* - ** Unit test for BulkEditReceiveValuesStep - *******************************************************************************/ -class BulkEditReceiveValuesStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate"); - stepInput.addValue("firstName", "Johnny"); - stepInput.addValue("email", null); - stepInput.addValue("birthDate", "1909-01-09"); - List records = TestUtils.queryTable(TestUtils.defineTablePerson().getName()); - stepInput.setRecords(records); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkEditReceiveValuesStep().run(stepInput, stepOutput); - - String valuesBeingUpdated = stepOutput.getValueString(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED); - assertThat(valuesBeingUpdated).matches("(?s).*First Name.*Johnny.*"); - assertThat(valuesBeingUpdated).matches("(?s).*Email will be cleared.*"); - assertThat(valuesBeingUpdated).matches("(?s).*Birth Date.*1909-01-09.*"); - - int count = 0; - for(QRecord record : stepOutput.getRecords()) - { - assertEquals("Johnny", record.getValueString("firstName")); - assertNull(record.getValue("email")); - // todo value utils needed in getValueDate... assertEquals(LocalDate.of(1909, 1, 9), record.getValueDate("birthDate")); - count++; - } - assertEquals(records.size(), count); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStepTest.java deleted file mode 100644 index 9b110db4..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStepTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - - -/******************************************************************************* - ** Unit test for BulkEditStoreRecordsStep - *******************************************************************************/ -class BulkEditStoreRecordsStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate"); - stepInput.addValue("firstName", "Johnny"); - stepInput.addValue("email", null); - stepInput.addValue("birthDate", "1909-01-09"); - List records = TestUtils.queryTable(TestUtils.defineTablePerson().getName()); - stepInput.setRecords(records); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkEditStoreRecordsStep().run(stepInput, stepOutput); - - assertRecordValues(stepOutput.getRecords()); - - // re-fetch the records, make sure they are updated. - // but since Mock backend doesn't actually update them, we can't do this.. - // todo - implement an in-memory backend, that would do this. - // List updatedRecords = TestUtils.queryTable(TestUtils.defineTablePerson().getName()); - // assertRecordValues(updatedRecords); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void assertRecordValues(List records) - { - for(QRecord record : records) - { - assertEquals("Johnny", record.getValueString("firstName")); - assertNull(record.getValue("email")); - // todo value utils needed in getValueDate... assertEquals(LocalDate.of(1909, 1, 9), record.getValueDate("birthDate")); - } - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java new file mode 100644 index 00000000..e1b666d7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java @@ -0,0 +1,145 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; + + +import java.util.List; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for full bulk edit process + *******************************************************************************/ +class BulkEditTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ////////////////////////////// + // insert some test records // + ////////////////////////////// + QInstance qInstance = TestUtils.defineInstance(); + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("email", "darin.kelkhoff@kingsrook.com"), + new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("email", "tim.chamberlain@kingsrook.com"), + new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("lastName", "Maes").withValue("email", "james.maes@kingsrook.com") + )); + + ////////////////////////////////// + // set up the run-process input // + ////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(qInstance); + runProcessInput.setSession(TestUtils.getMockSession()); + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkEdit"); + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, + new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2)))); + + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + runProcessInput.addValue(BulkEditTransformStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate"); + runProcessInput.addValue("firstName", "Johnny"); + runProcessInput.addValue("email", null); + runProcessInput.addValue("birthDate", "1909-01-09"); + + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("edit"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class); + + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + @SuppressWarnings("unchecked") + List processSummaryLines = (List) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + assertThat(processSummaryLines).hasSize(4); + assertThat(processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.OK))).hasSize(1); + List infoLines = processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.INFO)).collect(Collectors.toList()); + assertThat(infoLines).hasSize(3); + assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*First Name.*Johnny.*")); + assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*Email was cleared.*")); + assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*Birth Date.*1909-01-09.*")); + + ///////////////////////////////////////////////////////////////////////////////// + // query for the edited records - assert that id 1 & 2 were updated, 3 was not // + ///////////////////////////////////////////////////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals("Johnny", records.get(0).getValueString("firstName")); + assertEquals("Johnny", records.get(1).getValueString("firstName")); + assertEquals("James", records.get(2).getValueString("firstName")); + assertEquals("1909-01-09", records.get(0).getValueString("birthDate")); + assertEquals("1909-01-09", records.get(1).getValueString("birthDate")); + assertNull(records.get(2).getValueString("birthDate")); + assertNull(records.get(0).getValue("email")); + assertNull(records.get(1).getValue("email")); + assertEquals("james.maes@kingsrook.com", records.get(2).getValue("email")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java deleted file mode 100644 index 1b6d5659..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - - -/******************************************************************************* - ** Unit test for BulkInsertReceiveFileStep - *******************************************************************************/ -class BulkInsertReceiveFileStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - //////////////////////////////////////////////////////////////// - // create an uploaded file, similar to how an http server may // - //////////////////////////////////////////////////////////////// - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); - qUploadedFile.setFilename("test.csv"); - UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(key, qUploadedFile); - - //////////////////////////// - // setup and run the step // - //////////////////////////// - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkInsertReceiveFileStep().run(stepInput, stepOutput); - - List records = stepOutput.getRecords(); - assertEquals(2, records.size()); - assertEquals("John", records.get(0).getValueString("firstName")); - assertEquals("Jane", records.get(1).getValueString("firstName")); - assertNull(records.get(0).getValue("id")); - assertNull(records.get(1).getValue("id")); - - assertEquals(2, stepOutput.getValueInteger("noOfFileRows")); - } - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testBadFileType() throws QException - { - //////////////////////////////////////////////////////////////// - // create an uploaded file, similar to how an http server may // - //////////////////////////////////////////////////////////////// - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); // todo - this is NOT excel content... - qUploadedFile.setFilename("test.xslx"); - UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(key, qUploadedFile); - - //////////////////////////// - // setup and run the step // - //////////////////////////// - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - - assertThrows(QUserFacingException.class, () -> - { - new BulkInsertReceiveFileStep().run(stepInput, stepOutput); - }); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java deleted file mode 100644 index 2853abfc..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - - -/******************************************************************************* - ** Unit test for BulkInsertStoreRecordsStep - *******************************************************************************/ -class BulkInsertStoreRecordsStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - //////////////////////////////////////////////////////////////// - // create an uploaded file, similar to how an http server may // - //////////////////////////////////////////////////////////////// - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); - qUploadedFile.setFilename("test.csv"); - UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(key, qUploadedFile); - - //////////////////////////// - // setup and run the step // - //////////////////////////// - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkInsertStoreRecordsStep().run(stepInput, stepOutput); - - List records = stepOutput.getRecords(); - assertEquals(2, records.size()); - assertEquals("John", records.get(0).getValueString("firstName")); - assertEquals("Jane", records.get(1).getValueString("firstName")); - assertNotNull(records.get(0).getValue("id")); - assertNotNull(records.get(1).getValue("id")); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java new file mode 100644 index 00000000..c632ad0f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.state.StateType; +import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; +import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for full bulk insert process + *******************************************************************************/ +class BulkInsertTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + /////////////////////////////////////// + // make sure table is empty to start // + /////////////////////////////////////// + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); + + //////////////////////////////////////////////////////////////// + // create an uploaded file, similar to how an http server may // + //////////////////////////////////////////////////////////////// + QUploadedFile qUploadedFile = new QUploadedFile(); + qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); + qUploadedFile.setFilename("test.csv"); + UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); + TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile); + + RunProcessInput runProcessInput = new RunProcessInput(TestUtils.defineInstance()); + runProcessInput.setSession(TestUtils.getMockSession()); + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert"); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("upload"); + runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, uploadedFileKey); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class); + + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + //////////////////////////////////// + // query for the inserted records // + //////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals("John", records.get(0).getValueString("firstName")); + assertEquals("Jane", records.get(1).getValueString("firstName")); + assertNotNull(records.get(0).getValue("id")); + assertNotNull(records.get(1).getValue("id")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 80fc9cd6..a29afce5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarChart; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge; @@ -164,7 +163,6 @@ public class TestUtils - /******************************************************************************* ** *******************************************************************************/ @@ -207,6 +205,20 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static void insertRecords(QInstance qInstance, QTableMetaData table, List records) throws QException + { + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(table.getName()); + insertInput.setRecords(records); + new InsertAction().execute(insertInput); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java index feb81cd1..4e5a8e7f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java @@ -75,33 +75,56 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte // - else if there's a list, try to delete it, but upon error: // // - - do a single-delete for each entry in the list. // ///////////////////////////////////////////////////////////////////////////////// - try(Connection connection = getConnection(deleteInput)) + try { - /////////////////////////////////////////////////////////////////////////////////////////////// - // if there's a query filter, try to do a single-delete with that filter in the WHERE clause // - /////////////////////////////////////////////////////////////////////////////////////////////// - if(deleteInput.getQueryFilter() != null) + Connection connection; + boolean needToCloseConnection = false; + if(deleteInput.getTransaction() != null && deleteInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction) { - try - { - deleteInput.getAsyncJobCallback().updateStatus("Running bulk delete via query filter."); - deleteQueryFilter(connection, deleteInput, deleteOutput); - return (deleteOutput); - } - catch(Exception e) - { - deleteInput.getAsyncJobCallback().updateStatus("Error running bulk delete via filter. Fetching keys for individual deletes."); - LOG.info("Exception trying to delete by filter query. Moving on to deleting by id now."); - deleteInput.setPrimaryKeys(DeleteAction.getPrimaryKeysFromQueryFilter(deleteInput)); - } + LOG.debug("Using connection from updateInput [" + rdbmsTransaction.getConnection() + "]"); + connection = rdbmsTransaction.getConnection(); + } + else + { + connection = getConnection(deleteInput); + needToCloseConnection = true; } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // at this point, there either wasn't a query filter, or there was an error executing it (in which case, the query should // - // have been converted to a list of primary keys in the deleteInput). so, proceed now by deleting a list of pkeys. // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - deleteList(connection, deleteInput, deleteOutput); - return (deleteOutput); + try + { + /////////////////////////////////////////////////////////////////////////////////////////////// + // if there's a query filter, try to do a single-delete with that filter in the WHERE clause // + /////////////////////////////////////////////////////////////////////////////////////////////// + if(deleteInput.getQueryFilter() != null) + { + try + { + deleteInput.getAsyncJobCallback().updateStatus("Running bulk delete via query filter."); + deleteQueryFilter(connection, deleteInput, deleteOutput); + return (deleteOutput); + } + catch(Exception e) + { + deleteInput.getAsyncJobCallback().updateStatus("Error running bulk delete via filter. Fetching keys for individual deletes."); + LOG.info("Exception trying to delete by filter query. Moving on to deleting by id now."); + deleteInput.setPrimaryKeys(DeleteAction.getPrimaryKeysFromQueryFilter(deleteInput)); + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // at this point, there either wasn't a query filter, or there was an error executing it (in which case, the query should // + // have been converted to a list of primary keys in the deleteInput). so, proceed now by deleting a list of pkeys. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + deleteList(connection, deleteInput, deleteOutput); + return (deleteOutput); + } + finally + { + if(needToCloseConnection) + { + connection.close(); + } + } } catch(Exception e) { @@ -117,6 +140,13 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte private void deleteList(Connection connection, DeleteInput deleteInput, DeleteOutput deleteOutput) { List primaryKeys = deleteInput.getPrimaryKeys(); + if(primaryKeys.size() == 0) + { + ///////////////////////// + // noop - just return. // + ///////////////////////// + return; + } if(primaryKeys.size() == 1) { doDeleteOne(connection, deleteInput.getTable(), primaryKeys.get(0), deleteOutput);