From f1b2eef3b8a91c560a0005e85fcbbbf880c4bc6a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 26 Jul 2022 15:55:59 -0500 Subject: [PATCH] Feedback from code reviews --- .../core/actions/async/AsyncJobManager.java | 55 +++----- .../actions/reporting/CsvReportStreamer.java | 5 +- .../reporting/ExcelReportStreamer.java | 5 +- .../core/actions/reporting/RecordPipe.java | 11 +- .../core/actions/reporting/ReportAction.java | 124 ++++++++++++------ .../reporting/ReportStreamerInterface.java | 20 +-- .../core/actions/tables/DeleteAction.java | 1 + .../core/instances/QInstanceEnricher.java | 13 +- .../actions/processes/QUploadedFile.java | 2 + .../processes/RunBackendStepOutput.java | 25 +++- .../model/actions/reporting/ReportFormat.java | 18 ++- .../tables/query/QueryOutputRecordPipe.java | 12 +- .../model/metadata/fields/QFieldType.java | 3 + .../bulk/delete/BulkDeleteStoreStep.java | 16 ++- .../bulk/edit/BulkEditReceiveValuesStep.java | 26 +--- .../bulk/edit/BulkEditStoreRecordsStep.java | 4 +- .../bulk/edit/BulkEditUtils.java | 68 ++++++++++ .../bulk/insert/BulkInsertUtils.java | 24 +++- .../qqq/backend/core/state/StateType.java | 1 - .../actions/reporting/ReportActionTest.java | 66 +++++++++- .../bulk/delete/BulkDeleteStoreStepTest.java | 3 + .../insert/BulkInsertReceiveFileStepTest.java | 35 ++++- .../BulkInsertStoreRecordsStepTest.java | 2 +- .../qqq/backend/core/utils/TestUtils.java | 8 +- 24 files changed, 377 insertions(+), 170 deletions(-) create mode 100644 src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index 94106b7f..63766320 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -35,7 +35,6 @@ import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider; import com.kingsrook.qqq.backend.core.state.StateProviderInterface; import com.kingsrook.qqq.backend.core.state.StateType; import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; -import com.kingsrook.qqq.backend.core.utils.SleepUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -78,6 +77,11 @@ public class AsyncJobManager return (runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus)); }); + if(timeout == 0) + { + throw (new JobGoingAsyncException(uuidAndTypeStateKey.getUuid().toString())); + } + T result = future.get(timeout, timeUnit); return (result); } @@ -97,7 +101,7 @@ public class AsyncJobManager /******************************************************************************* ** Start a job, and always, just get back the job UUID. *******************************************************************************/ - public String startJob(AsyncJob asyncJob) + public String startJob(AsyncJob asyncJob) throws QException { return (startJob("Anonymous", asyncJob)); } @@ -107,17 +111,17 @@ public class AsyncJobManager /******************************************************************************* ** Start a job, and always, just get back the job UUID. *******************************************************************************/ - public String startJob(String jobName, AsyncJob asyncJob) + public String startJob(String jobName, AsyncJob asyncJob) throws QException { - UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS); - AsyncJobStatus asyncJobStatus = new AsyncJobStatus(); - asyncJobStatus.setState(AsyncJobState.RUNNING); - getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); - - // todo - refactor, share - CompletableFuture.supplyAsync(() -> runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus)); - - return (uuidAndTypeStateKey.getUuid().toString()); + try + { + startJob(jobName, 0, TimeUnit.MILLISECONDS, asyncJob); + throw (new QException("Job was expected to go asynchronous, but did not")); + } + catch(JobGoingAsyncException jgae) + { + return (jgae.getJobUUID()); + } } @@ -179,31 +183,4 @@ public class AsyncJobManager // return TempFileStateProvider.getInstance(); } - - - /******************************************************************************* - ** - *******************************************************************************/ - public AsyncJobStatus waitForJob(String jobUUID) throws QException - { - AsyncJobState asyncJobState = AsyncJobState.RUNNING; - AsyncJobStatus asyncJobStatus = null; - while(asyncJobState.equals(AsyncJobState.RUNNING)) - { - LOG.info("Sleeping, waiting on job [" + jobUUID + "]"); - SleepUtils.sleep(100, TimeUnit.MILLISECONDS); - - Optional optionalAsyncJobStatus = getJobStatus(jobUUID); - if(optionalAsyncJobStatus.isEmpty()) - { - // todo - ... maybe some version of try-again? - throw (new QException("Could not get status of report query job [" + jobUUID + "]")); - } - asyncJobStatus = optionalAsyncJobStatus.get(); - asyncJobState = asyncJobStatus.getState(); - } - - return (asyncJobStatus); - } - } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java index 34f2dab4..b6f73caf 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java @@ -65,14 +65,13 @@ public class CsvReportStreamer implements ReportStreamerInterface ** *******************************************************************************/ @Override - public void start(ReportInput reportInput) throws QReportingException + public void start(ReportInput reportInput, List fields) throws QReportingException { this.reportInput = reportInput; + this.fields = fields; table = reportInput.getTable(); outputStream = this.reportInput.getReportOutputStream(); - fields = setupFieldList(table, reportInput); - writeReportHeaderRow(); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java index 2d048b27..004fe8a8 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java @@ -72,17 +72,16 @@ public class ExcelReportStreamer implements ReportStreamerInterface ** *******************************************************************************/ @Override - public void start(ReportInput reportInput) throws QReportingException + public void start(ReportInput reportInput, List fields) throws QReportingException { this.reportInput = reportInput; + this.fields = fields; table = reportInput.getTable(); outputStream = this.reportInput.getReportOutputStream(); workbook = new Workbook(outputStream, "QQQ", null); worksheet = workbook.newWorksheet("Sheet 1"); - fields = setupFieldList(table, reportInput); - writeReportHeaderRow(); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java index b03ba662..cb8e3b6d 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -22,10 +22,9 @@ package com.kingsrook.qqq.backend.core.actions.reporting; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; -import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -35,16 +34,16 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; *******************************************************************************/ public class RecordPipe { - private Queue queue = new ArrayDeque<>(); - + private ArrayBlockingQueue queue = new ArrayBlockingQueue<>(10_000); /******************************************************************************* ** Add a record to the pipe + ** Returns true iff the record fit in the pipe; false if the pipe is currently full. *******************************************************************************/ - public void addRecord(QRecord record) + public boolean addRecord(QRecord record) { - queue.add(record); + return (queue.offer(record)); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java index 12085947..a63ae0d4 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -66,9 +67,13 @@ public class ReportAction { private static final Logger LOG = LogManager.getLogger(ReportAction.class); - private boolean preExecuteRan = false; + private boolean preExecuteRan = false; private Integer countFromPreExecute = null; + private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000; + private static final int MAX_SLEEP_MS = 1000; + private static final int INIT_SLEEP_MS = 10; + /******************************************************************************* @@ -137,7 +142,6 @@ public class ReportAction QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); QBackendModuleInterface backendModule = qBackendModuleDispatcher.getQBackendModule(reportInput.getBackend()); - QTableMetaData table = reportInput.getTable(); ////////////////////////// // set up a query input // @@ -160,64 +164,73 @@ public class ReportAction //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ReportFormat reportFormat = reportInput.getReportFormat(); ReportStreamerInterface reportStreamer = reportFormat.newReportStreamer(); - reportStreamer.start(reportInput); + reportStreamer.start(reportInput, getFields(reportInput)); ////////////////////////////////////////// // run the query action as an async job // ////////////////////////////////////////// AsyncJobManager asyncJobManager = new AsyncJobManager(); - String jobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput))); - LOG.info("Started query job [" + jobUUID + "] for report"); + String queryJobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput))); + LOG.info("Started query job [" + queryJobUUID + "] for report"); - AsyncJobState asyncJobState = AsyncJobState.RUNNING; - AsyncJobStatus asyncJobStatus = null; - int nextSleepMillis = 10; - long recordCount = 0; - while(asyncJobState.equals(AsyncJobState.RUNNING)) + AsyncJobState queryJobState = AsyncJobState.RUNNING; + AsyncJobStatus asyncJobStatus = null; + + long recordCount = 0; + int nextSleepMillis = INIT_SLEEP_MS; + long lastReceivedRecordsAt = System.currentTimeMillis(); + long reportStartTime = System.currentTimeMillis(); + + while(queryJobState.equals(AsyncJobState.RUNNING)) { - int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe); - recordCount += recordsConsumed; - - if(countFromPreExecute != null) + if(recordPipe.countAvailableRecords() == 0) { - LOG.info(String.format("Processed %,d of %,d records so far", recordCount, countFromPreExecute)); + /////////////////////////////////////////////////////////// + // if the pipe is empty, sleep to let the producer work. // + // todo - smarter sleep? like get notified vs. sleep? // + /////////////////////////////////////////////////////////// + LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); + SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); + nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS); + + long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt; + if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS) + { + throw (new QReportingException("Query action appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago).")); + } } else { - LOG.info(String.format("Processed %,d records so far", recordCount)); + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + lastReceivedRecordsAt = System.currentTimeMillis(); + nextSleepMillis = INIT_SLEEP_MS; + + int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe); + recordCount += recordsConsumed; + + LOG.info(countFromPreExecute != null + ? String.format("Processed %,d of %,d records so far", recordCount, countFromPreExecute) + : String.format("Processed %,d records so far", recordCount)); } - if(recordsConsumed == 0) - { - //////////////////////////////////////////////////////////////////////////// - // do we need to sleep to let the producer work? // - // todo - smarter sleep? like get notified vs. sleep? eventually a fail? // - //////////////////////////////////////////////////////////////////////////// - LOG.info("Read 0 records from pipe, sleeping to give producer a chance to work"); - SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); - while(recordPipe.countAvailableRecords() == 0) - { - nextSleepMillis = Math.min(nextSleepMillis * 2, 1000); - LOG.info("Still no records in the pipe, so sleeping more [" + nextSleepMillis + "]ms"); - SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); - } - } - - nextSleepMillis = 10; - - Optional optionalAsyncJobStatus = asyncJobManager.getJobStatus(jobUUID); + //////////////////////////////////// + // refresh the query job's status // + //////////////////////////////////// + Optional optionalAsyncJobStatus = asyncJobManager.getJobStatus(queryJobUUID); if(optionalAsyncJobStatus.isEmpty()) { ///////////////////////////////////////////////// // todo - ... maybe some version of try-again? // ///////////////////////////////////////////////// - throw (new QException("Could not get status of report query job [" + jobUUID + "]")); + throw (new QException("Could not get status of report query job [" + queryJobUUID + "]")); } asyncJobStatus = optionalAsyncJobStatus.get(); - asyncJobState = asyncJobStatus.getState(); + queryJobState = asyncJobStatus.getState(); } - LOG.info("Query job [" + jobUUID + "] for report completed with status: " + asyncJobStatus); + LOG.info("Query job [" + queryJobUUID + "] for report completed with status: " + asyncJobStatus); /////////////////////////////////////////////////// // send the final records to the report streamer // @@ -225,6 +238,12 @@ public class ReportAction int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe); recordCount += recordsConsumed; + long reportEndTime = System.currentTimeMillis(); + LOG.info((countFromPreExecute != null + ? String.format("Processed %,d of %,d records", recordCount, countFromPreExecute) + : String.format("Processed %,d records", recordCount)) + + String.format(" at end of report in %,d ms (%.2f records/second).", (reportEndTime - reportStartTime), 1000d * (recordCount / (.001d + (reportEndTime - reportStartTime))))); + ////////////////////////////////////////////////////////////////// // Critical: we must close the stream here as our final action // ////////////////////////////////////////////////////////////////// @@ -247,11 +266,40 @@ public class ReportAction + /******************************************************************************* + ** + *******************************************************************************/ + private List getFields(ReportInput reportInput) + { + QTableMetaData table = reportInput.getTable(); + if(reportInput.getFieldNames() != null) + { + return (reportInput.getFieldNames().stream().map(table::getField).toList()); + } + else + { + return (new ArrayList<>(table.getFields().values())); + } + } + + + /******************************************************************************* ** *******************************************************************************/ private void verifyCountUnderMax(ReportInput reportInput, QBackendModuleInterface backendModule, ReportFormat reportFormat) throws QException { + if(reportFormat.getMaxCols() != null) + { + List fields = getFields(reportInput); + if (fields.size() > reportFormat.getMaxCols()) + { + throw (new QUserFacingException("The requested report would include more columns (" + + String.format("%,d", fields.size()) + ") than the maximum allowed (" + + String.format("%,d", reportFormat.getMaxCols()) + ") for the selected file format (" + reportFormat + ").")); + } + } + if(reportFormat.getMaxRows() != null) { if(reportInput.getLimit() == null || reportInput.getLimit() > reportFormat.getMaxRows()) diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java index fe65fce3..b75610c7 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java @@ -22,12 +22,10 @@ package com.kingsrook.qqq.backend.core.actions.reporting; -import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; /******************************************************************************* @@ -38,7 +36,7 @@ public interface ReportStreamerInterface /******************************************************************************* ** Called once, before any rows are available. Meant to write a header, for example. *******************************************************************************/ - void start(ReportInput reportInput) throws QReportingException; + void start(ReportInput reportInput, List fields) throws QReportingException; /******************************************************************************* ** Called as records flow into the pipe. @@ -50,20 +48,4 @@ public interface ReportStreamerInterface *******************************************************************************/ void finish() throws QReportingException; - /******************************************************************************* - ** (Ideally, protected) method used within report streamer implementations, to - ** map field names from reportInput into list of fieldMetaData. - *******************************************************************************/ - default List setupFieldList(QTableMetaData table, ReportInput reportInput) - { - if(reportInput.getFieldNames() != null) - { - return (reportInput.getFieldNames().stream().map(table::getField).toList()); - } - else - { - return (new ArrayList<>(table.getFields().values())); - } - } - } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index 7b3ed6d8..21ce7bee 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -77,6 +77,7 @@ public class DeleteAction LOG.info("0 primaryKeys found. Returning with no-op"); DeleteOutput deleteOutput = new DeleteOutput(); deleteOutput.setRecordsWithErrors(new ArrayList<>()); + deleteOutput.setDeletedRecordCount(0); return (deleteOutput); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index a23f5255..569ce603 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -246,14 +246,12 @@ public class QInstanceEnricher .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true)) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) - .withValue("text", "Upload a CSV or XLSX file with the following columns: " + fieldsForHelpText)); + // .withValue("text", "Upload a CSV or XLSX file with the following columns: " + fieldsForHelpText)); + .withValue("text", "Upload a CSV file with the following columns: " + fieldsForHelpText)); QBackendStepMetaData receiveFileStep = new QBackendStepMetaData() .withName("receiveFile") .withCode(new QCodeReference(BulkInsertReceiveFileStep.class)) - .withInputData(new QFunctionInputMetaData() - // todo - our upload file as a field? problem is, its type... - .withFieldList(List.of())) .withOutputMetaData(new QFunctionOutputMetaData() .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER)))); @@ -268,9 +266,6 @@ public class QInstanceEnricher QBackendStepMetaData storeStep = new QBackendStepMetaData() .withName("storeRecords") .withCode(new QCodeReference(BulkInsertStoreRecordsStep.class)) - .withInputData(new QFunctionInputMetaData() - // todo - our upload file as a field? problem is, its type... - .withFieldList(List.of())) .withOutputMetaData(new QFunctionOutputMetaData() .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER)))); @@ -342,15 +337,13 @@ public class QInstanceEnricher QBackendStepMetaData storeStep = new QBackendStepMetaData() .withName("storeRecords") .withCode(new QCodeReference(BulkEditStoreRecordsStep.class)) - .withInputData(new QFunctionInputMetaData() - // todo - our upload file as a field? problem is, its type... - .withFieldList(List.of())) .withOutputMetaData(new QFunctionOutputMetaData() .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER)))); QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData() .withName("results") .withRecordListFields(new ArrayList<>(table.getFields().values())) + .withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING)) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) .withValue("text", "The records below have been updated.")); diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/QUploadedFile.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/QUploadedFile.java index 7fdf8347..81124224 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/QUploadedFile.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/QUploadedFile.java @@ -30,6 +30,8 @@ import java.io.Serializable; *******************************************************************************/ public class QUploadedFile implements Serializable { + public static final String DEFAULT_UPLOADED_FILE_FIELD_NAME = "uploadedFileKey"; + private String filename; private byte[] bytes; diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java index c11df1fb..a6460bff 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes; import java.io.Serializable; +import java.math.BigDecimal; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; @@ -204,7 +205,7 @@ public class RunBackendStepOutput extends AbstractActionOutput *******************************************************************************/ public String getValueString(String fieldName) { - return ((String) getValue(fieldName)); + return (ValueUtils.getValueAsString(getValue(fieldName))); } @@ -218,4 +219,26 @@ public class RunBackendStepOutput extends AbstractActionOutput return (ValueUtils.getValueAsInteger(getValue(fieldName))); } + + + /******************************************************************************* + ** Getter for a single field's value + ** + *******************************************************************************/ + public Boolean getValueBoolean(String fieldName) + { + return (ValueUtils.getValueAsBoolean(getValue(fieldName))); + } + + + + /******************************************************************************* + ** Getter for a single field's value + ** + *******************************************************************************/ + public BigDecimal getValueBigDecimal(String fieldName) + { + return (ValueUtils.getValueAsBigDecimal(getValue(fieldName))); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java index fd874601..045229a4 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java @@ -37,11 +37,12 @@ import org.dhatim.fastexcel.Worksheet; *******************************************************************************/ public enum ReportFormat { - XLSX(Worksheet.MAX_ROWS, ExcelReportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), - CSV(null, CsvReportStreamer::new, "text/csv"); + XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelReportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), + CSV(null, null, CsvReportStreamer::new, "text/csv"); private final Integer maxRows; + private final Integer maxCols; private final String mimeType; private final Supplier streamerConstructor; @@ -51,9 +52,10 @@ public enum ReportFormat /******************************************************************************* ** *******************************************************************************/ - ReportFormat(Integer maxRows, Supplier streamerConstructor, String mimeType) + ReportFormat(Integer maxRows, Integer maxCols, Supplier streamerConstructor, String mimeType) { this.maxRows = maxRows; + this.maxCols = maxCols; this.mimeType = mimeType; this.streamerConstructor = streamerConstructor; } @@ -92,6 +94,16 @@ public enum ReportFormat } + /******************************************************************************* + ** Getter for maxCols + ** + *******************************************************************************/ + public Integer getMaxCols() + { + return maxCols; + } + + /******************************************************************************* ** Getter for mimeType diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java index 52776550..9e65ade9 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java @@ -58,8 +58,16 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface @Override public void addRecord(QRecord record) { - recordPipe.addRecord(record); - blockIfPipeIsTooFull(); + if(!recordPipe.addRecord(record)) + { + do + { + LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); + SleepUtils.sleep(10, TimeUnit.MILLISECONDS); + } + while(!recordPipe.addRecord(record)); + LOG.debug("Done blocking."); + } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index 3f4f47fa..324e8991 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -42,6 +42,9 @@ public enum QFieldType HTML, PASSWORD, BLOB; + /////////////////////////////////////////////////////////////////////// + // keep these values in sync with QFieldType.ts in qqq-frontend-core // + /////////////////////////////////////////////////////////////////////// diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java index 8d9c2a66..c8cf5f11 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java @@ -24,7 +24,9 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; import java.io.IOException; import java.io.Serializable; +import java.util.Collections; import java.util.List; +import java.util.Objects; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -33,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -43,6 +46,9 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class BulkDeleteStoreStep implements BackendStep { + public static final String ERROR_COUNT = "errorCount"; + + /******************************************************************************* ** @@ -50,7 +56,7 @@ public class BulkDeleteStoreStep implements BackendStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records in database..."); + runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records..."); runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal(); DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance()); @@ -78,12 +84,16 @@ public class BulkDeleteStoreStep implements BackendStep .toList(); deleteInput.setPrimaryKeys(primaryKeyList); } + else + { + throw (new QException("Missing required inputs (queryFilterJSON or record list)")); + } DeleteAction deleteAction = new DeleteAction(); DeleteOutput deleteOutput = deleteAction.execute(deleteInput); - // todo - something with the output!! - deleteOutput.getRecordsWithErrors(); + List recordsWithErrors = Objects.requireNonNullElse(deleteOutput.getRecordsWithErrors(), Collections.emptyList()); + runBackendStepOutput.addValue(ERROR_COUNT, recordsWithErrors.size()); runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java index bc8a896d..8253999a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java @@ -23,16 +23,11 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -68,26 +63,7 @@ public class BulkEditReceiveValuesStep implements BackendStep } } - ///////////////////////////////////////////////////////////////////// - // build the string to show the user what fields are being changed // - ///////////////////////////////////////////////////////////////////// - List valuesBeingUpdated = new ArrayList<>(); - QTableMetaData table = runBackendStepInput.getTable(); - for(String fieldName : enabledFields) - { - String label = table.getField(fieldName).getLabel(); - Serializable value = runBackendStepInput.getValue(fieldName); - - if(StringUtils.hasContent(ValueUtils.getValueAsString(value))) - { - valuesBeingUpdated.add(label + " will be set to: " + value); - } - else - { - valuesBeingUpdated.add(label + " will be cleared out."); - } - } - runBackendStepOutput.addValue(FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated)); + BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "will be"); runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java index 5928b801..f52a7335 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java @@ -69,7 +69,7 @@ public class BulkEditStoreRecordsStep implements BackendStep } } - runBackendStepInput.getAsyncJobCallback().updateStatus("Updating database..."); + runBackendStepInput.getAsyncJobCallback().updateStatus("Storing updated records..."); runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal(); UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); @@ -83,6 +83,8 @@ public class BulkEditStoreRecordsStep implements BackendStep UpdateOutput updateOutput = updateAction.execute(updateInput); runBackendStepOutput.setRecords(updateOutput.getRecords()); + + BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "was"); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java new file mode 100644 index 00000000..71eeb69f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Utility methods used for Bulk Edit steps + *******************************************************************************/ +public class BulkEditUtils +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static void setFieldValuesBeingUpdated(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, String[] enabledFields, String verb) + { + ///////////////////////////////////////////////////////////////////// + // build the string to show the user what fields are being changed // + ///////////////////////////////////////////////////////////////////// + List valuesBeingUpdated = new ArrayList<>(); + QTableMetaData table = runBackendStepInput.getTable(); + for(String fieldName : enabledFields) + { + String label = table.getField(fieldName).getLabel(); + Serializable value = runBackendStepInput.getValue(fieldName); + + if(StringUtils.hasContent(ValueUtils.getValueAsString(value))) + { + valuesBeingUpdated.add(label + " " + verb + " set to: " + value); + } + else + { + valuesBeingUpdated.add(label + " " + verb + " cleared out"); + } + } + runBackendStepOutput.addValue(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated)); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java index f5684a72..4d4182bb 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java @@ -23,10 +23,12 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; @@ -47,28 +49,36 @@ public class BulkInsertUtils *******************************************************************************/ static List getQRecordsFromFile(RunBackendStepInput runBackendStepInput) throws QException { - AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue("uploadedFileKey"); + AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME); Optional optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey); if(optionalUploadedFile.isEmpty()) { throw (new QException("Could not find uploaded file")); } - byte[] bytes = optionalUploadedFile.get().getBytes(); + byte[] bytes = optionalUploadedFile.get().getBytes(); + String fileName = optionalUploadedFile.get().getFilename(); ///////////////////////////////////////////////////// // let the user specify field labels instead names // ///////////////////////////////////////////////////// - QTableMetaData table = runBackendStepInput.getTable(); - QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping(); + QTableMetaData table = runBackendStepInput.getTable(); + String tableName = runBackendStepInput.getTableName(); + QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping(); for(Map.Entry entry : table.getFields().entrySet()) { mapping.addMapping(entry.getKey(), entry.getValue().getLabel()); } - // todo - sniff out file type... - String tableName = runBackendStepInput.getTableName(); - List qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping); + List qRecords; + if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) + { + qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping); + } + else + { + throw (new QUserFacingException("Unsupported file type.")); + } //////////////////////////////////////////////// // remove values from any non-editable fields // diff --git a/src/main/java/com/kingsrook/qqq/backend/core/state/StateType.java b/src/main/java/com/kingsrook/qqq/backend/core/state/StateType.java index 47c26a71..841045bc 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/state/StateType.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/state/StateType.java @@ -32,6 +32,5 @@ public enum StateType { PROCESS_STATUS, ASYNC_JOB_STATUS, - ASYNC_JOB_RESULT, UPLOADED_FILE } diff --git a/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java b/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java index c356f7ec..f29a62fa 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java @@ -34,7 +34,9 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.apache.commons.io.FileUtils; @@ -70,6 +72,27 @@ class ReportActionTest + /******************************************************************************* + ** This test runs for more records, to stress more of the pipe-filling and + ** other bits of the ReportAction. + *******************************************************************************/ + @Test + public void testBigger() throws Exception + { + // int recordCount = 2_000_000; // to really stress locally, use this. + int recordCount = 200_000; + String filename = "/tmp/ReportActionTest.csv"; + + runReport(recordCount, filename, ReportFormat.CSV, false); + + File file = new File(filename); + List fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name()); + assertEquals(recordCount + 1, fileLines.size()); + assertTrue(file.delete()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -142,12 +165,49 @@ class ReportActionTest ReportInput reportInput = new ReportInput(TestUtils.defineInstance(), TestUtils.getMockSession()); reportInput.setTableName("person"); - //////////////////////////////////////////////////////////////// - // use xlsx, which has a max-rows limit, to verify that code. // - //////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // use xlsx, which has a max-rows limit, to verify that code runs, but doesn't throw when there aren't too many rows // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// reportInput.setReportFormat(ReportFormat.XLSX); new ReportAction().preExecute(reportInput); + + //////////////////////////////////////////////////////////////////////////// + // nothing to assert - but if preExecute throws, then the test will fail. // + //////////////////////////////////////////////////////////////////////////// + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTooManyColumns() throws QException + { + QTableMetaData wideTable = new QTableMetaData() + .withName("wide") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME); + for(int i = 0; i < ReportFormat.XLSX.getMaxCols() + 1; i++) + { + wideTable.addField(new QFieldMetaData("field" + i, QFieldType.STRING)); + } + + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(wideTable); + + ReportInput reportInput = new ReportInput(qInstance, TestUtils.getMockSession()); + reportInput.setTableName("wide"); + + //////////////////////////////////////////////////////////////// + // use xlsx, which has a max-cols limit, to verify that code. // + //////////////////////////////////////////////////////////////// + reportInput.setReportFormat(ReportFormat.XLSX); + + assertThrows(QUserFacingException.class, () -> + { + new ReportAction().preExecute(reportInput); + }); } } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java index db642bc4..483e2edf 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -50,6 +51,7 @@ class BulkDeleteStoreStepTest RunBackendStepOutput stepOutput = new RunBackendStepOutput(); new BulkDeleteStoreStep().run(stepInput, stepOutput); + assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT)); } @@ -66,6 +68,7 @@ class BulkDeleteStoreStepTest RunBackendStepOutput stepOutput = new RunBackendStepOutput(); new BulkDeleteStoreStep().run(stepInput, stepOutput); + assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT)); } } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java index df53ac98..1b6d5659 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; @@ -63,7 +64,7 @@ class BulkInsertReceiveFileStepTest RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); stepInput.setSession(TestUtils.getMockSession()); stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue("uploadedFileKey", key); + stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); RunBackendStepOutput stepOutput = new RunBackendStepOutput(); new BulkInsertReceiveFileStep().run(stepInput, stepOutput); @@ -78,4 +79,36 @@ class BulkInsertReceiveFileStepTest assertEquals(2, stepOutput.getValueInteger("noOfFileRows")); } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadFileType() throws QException + { + //////////////////////////////////////////////////////////////// + // create an uploaded file, similar to how an http server may // + //////////////////////////////////////////////////////////////// + QUploadedFile qUploadedFile = new QUploadedFile(); + qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); // todo - this is NOT excel content... + qUploadedFile.setFilename("test.xslx"); + UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); + TempFileStateProvider.getInstance().put(key, qUploadedFile); + + //////////////////////////// + // setup and run the step // + //////////////////////////// + RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); + stepInput.setSession(TestUtils.getMockSession()); + stepInput.setTableName(TestUtils.defineTablePerson().getName()); + stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); + + RunBackendStepOutput stepOutput = new RunBackendStepOutput(); + + assertThrows(QUserFacingException.class, () -> + { + new BulkInsertReceiveFileStep().run(stepInput, stepOutput); + }); + } + } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java index 7e0b4860..2853abfc 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java @@ -64,7 +64,7 @@ class BulkInsertStoreRecordsStepTest RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); stepInput.setSession(TestUtils.getMockSession()); stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue("uploadedFileKey", key); + stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); RunBackendStepOutput stepOutput = new RunBackendStepOutput(); new BulkInsertStoreRecordsStep().run(stepInput, stepOutput); diff --git a/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 379caad5..6bae6380 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -335,7 +335,7 @@ public class TestUtils public static String getPersonCsvHeader() { return (""" - "id","createDate","modifyDate","firstName","lastName","birthDate","email"\r + "id","createDate","modifyDate","firstName","lastName","birthDate","email" """); } @@ -347,7 +347,7 @@ public class TestUtils public static String getPersonCsvHeaderUsingLabels() { return (""" - "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"\r + "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email" """); } @@ -359,7 +359,7 @@ public class TestUtils public static String getPersonCsvRow1() { return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"\r + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com" """); } @@ -371,7 +371,7 @@ public class TestUtils public static String getPersonCsvRow2() { return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"\r + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com" """); }