QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feedback from code reviews

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-26 15:55:59 -05:00
parent cfa9cec930
commit f1b2eef3b8
24 changed files with 377 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java
View File

@ -35,7 +35,6 @@ import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -78,6 +77,11 @@ public class AsyncJobManager
return (runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
});
if(timeout == 0)
{
throw (new JobGoingAsyncException(uuidAndTypeStateKey.getUuid().toString()));
}
T result = future.get(timeout, timeUnit);
return (result);
}
@ -97,7 +101,7 @@ public class AsyncJobManager
/*******************************************************************************
** Start a job, and always, just get back the job UUID.
*******************************************************************************/
public <T extends Serializable> String startJob(AsyncJob<T> asyncJob)
public <T extends Serializable> String startJob(AsyncJob<T> asyncJob) throws QException
{
return (startJob("Anonymous", asyncJob));
}
@ -107,17 +111,17 @@ public class AsyncJobManager
/*******************************************************************************
** Start a job, and always, just get back the job UUID.
*******************************************************************************/
public <T extends Serializable> String startJob(String jobName, AsyncJob<T> asyncJob)
public <T extends Serializable> String startJob(String jobName, AsyncJob<T> asyncJob) throws QException
{
UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS);
AsyncJobStatus asyncJobStatus = new AsyncJobStatus();
asyncJobStatus.setState(AsyncJobState.RUNNING);
getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
// todo - refactor, share
CompletableFuture.supplyAsync(() -> runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
return (uuidAndTypeStateKey.getUuid().toString());
try
{
startJob(jobName, 0, TimeUnit.MILLISECONDS, asyncJob);
throw (new QException("Job was expected to go asynchronous, but did not"));
}
catch(JobGoingAsyncException jgae)
{
return (jgae.getJobUUID());
}
}
@ -179,31 +183,4 @@ public class AsyncJobManager
// return TempFileStateProvider.getInstance();
}
/*******************************************************************************
**
*******************************************************************************/
public AsyncJobStatus waitForJob(String jobUUID) throws QException
{
AsyncJobState asyncJobState = AsyncJobState.RUNNING;
AsyncJobStatus asyncJobStatus = null;
while(asyncJobState.equals(AsyncJobState.RUNNING))
{
LOG.info("Sleeping, waiting on job [" + jobUUID + "]");
SleepUtils.sleep(100, TimeUnit.MILLISECONDS);
Optional<AsyncJobStatus> optionalAsyncJobStatus = getJobStatus(jobUUID);
if(optionalAsyncJobStatus.isEmpty())
{
// todo - ... maybe some version of try-again?
throw (new QException("Could not get status of report query job [" + jobUUID + "]"));
}
asyncJobStatus = optionalAsyncJobStatus.get();
asyncJobState = asyncJobStatus.getState();
}
return (asyncJobStatus);
}
}

5
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java
View File

@ -65,14 +65,13 @@ public class CsvReportStreamer implements ReportStreamerInterface
**
*******************************************************************************/
@Override
public void start(ReportInput reportInput) throws QReportingException
public void start(ReportInput reportInput, List<QFieldMetaData> fields) throws QReportingException
{
this.reportInput = reportInput;
this.fields = fields;
table = reportInput.getTable();
outputStream = this.reportInput.getReportOutputStream();
fields = setupFieldList(table, reportInput);
writeReportHeaderRow();
}

5
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java
View File

@ -72,17 +72,16 @@ public class ExcelReportStreamer implements ReportStreamerInterface
**
*******************************************************************************/
@Override
public void start(ReportInput reportInput) throws QReportingException
public void start(ReportInput reportInput, List<QFieldMetaData> fields) throws QReportingException
{
this.reportInput = reportInput;
this.fields = fields;
table = reportInput.getTable();
outputStream = this.reportInput.getReportOutputStream();
workbook = new Workbook(outputStream, "QQQ", null);
worksheet = workbook.newWorksheet("Sheet 1");
fields = setupFieldList(table, reportInput);
writeReportHeaderRow();
}

11
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java
View File

@ -22,10 +22,9 @@
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ArrayBlockingQueue;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -35,16 +34,16 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
*******************************************************************************/
public class RecordPipe
{
private Queue<QRecord> queue = new ArrayDeque<>();
private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(10_000);
/*******************************************************************************
** Add a record to the pipe
** Returns true iff the record fit in the pipe; false if the pipe is currently full.
*******************************************************************************/
public void addRecord(QRecord record)
public boolean addRecord(QRecord record)
{
queue.add(record);
return (queue.offer(record));
}

124
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java
View File

@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@ -66,9 +67,13 @@ public class ReportAction
{
private static final Logger LOG = LogManager.getLogger(ReportAction.class);
private boolean preExecuteRan = false;
private boolean preExecuteRan = false;
private Integer countFromPreExecute = null;
private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000;
private static final int MAX_SLEEP_MS = 1000;
private static final int INIT_SLEEP_MS = 10;
/*******************************************************************************
@ -137,7 +142,6 @@ public class ReportAction
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface backendModule = qBackendModuleDispatcher.getQBackendModule(reportInput.getBackend());
QTableMetaData table = reportInput.getTable();
//////////////////////////
// set up a query input //
@ -160,64 +164,73 @@ public class ReportAction
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ReportFormat reportFormat = reportInput.getReportFormat();
ReportStreamerInterface reportStreamer = reportFormat.newReportStreamer();
reportStreamer.start(reportInput);
reportStreamer.start(reportInput, getFields(reportInput));
//////////////////////////////////////////
// run the query action as an async job //
//////////////////////////////////////////
AsyncJobManager asyncJobManager = new AsyncJobManager();
String jobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput)));
LOG.info("Started query job [" + jobUUID + "] for report");
String queryJobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput)));
LOG.info("Started query job [" + queryJobUUID + "] for report");
AsyncJobState asyncJobState = AsyncJobState.RUNNING;
AsyncJobStatus asyncJobStatus = null;
int nextSleepMillis = 10;
long recordCount = 0;
while(asyncJobState.equals(AsyncJobState.RUNNING))
AsyncJobState queryJobState = AsyncJobState.RUNNING;
AsyncJobStatus asyncJobStatus = null;
long recordCount = 0;
int nextSleepMillis = INIT_SLEEP_MS;
long lastReceivedRecordsAt = System.currentTimeMillis();
long reportStartTime = System.currentTimeMillis();
while(queryJobState.equals(AsyncJobState.RUNNING))
{
int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
recordCount += recordsConsumed;
if(countFromPreExecute != null)
if(recordPipe.countAvailableRecords() == 0)
{
LOG.info(String.format("Processed %,d of %,d records so far", recordCount, countFromPreExecute));
///////////////////////////////////////////////////////////
// if the pipe is empty, sleep to let the producer work. //
// todo - smarter sleep? like get notified vs. sleep? //
///////////////////////////////////////////////////////////
LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work");
SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS);
long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt;
if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS)
{
throw (new QReportingException("Query action appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago)."));
}
}
else
{
LOG.info(String.format("Processed %,d records so far", recordCount));
////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. //
////////////////////////////////////////////////////////////////////////////////////////////////////////
lastReceivedRecordsAt = System.currentTimeMillis();
nextSleepMillis = INIT_SLEEP_MS;
int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
recordCount += recordsConsumed;
LOG.info(countFromPreExecute != null
? String.format("Processed %,d of %,d records so far", recordCount, countFromPreExecute)
: String.format("Processed %,d records so far", recordCount));
}
if(recordsConsumed == 0)
{
////////////////////////////////////////////////////////////////////////////
// do we need to sleep to let the producer work? //
// todo - smarter sleep? like get notified vs. sleep? eventually a fail? //
////////////////////////////////////////////////////////////////////////////
LOG.info("Read 0 records from pipe, sleeping to give producer a chance to work");
SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
while(recordPipe.countAvailableRecords() == 0)
{
nextSleepMillis = Math.min(nextSleepMillis * 2, 1000);
LOG.info("Still no records in the pipe, so sleeping more [" + nextSleepMillis + "]ms");
SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
}
}
nextSleepMillis = 10;
Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(jobUUID);
////////////////////////////////////
// refresh the query job's status //
////////////////////////////////////
Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(queryJobUUID);
if(optionalAsyncJobStatus.isEmpty())
{
/////////////////////////////////////////////////
// todo - ... maybe some version of try-again? //
/////////////////////////////////////////////////
throw (new QException("Could not get status of report query job [" + jobUUID + "]"));
throw (new QException("Could not get status of report query job [" + queryJobUUID + "]"));
}
asyncJobStatus = optionalAsyncJobStatus.get();
asyncJobState = asyncJobStatus.getState();
queryJobState = asyncJobStatus.getState();
}
LOG.info("Query job [" + jobUUID + "] for report completed with status: " + asyncJobStatus);
LOG.info("Query job [" + queryJobUUID + "] for report completed with status: " + asyncJobStatus);
///////////////////////////////////////////////////
// send the final records to the report streamer //
@ -225,6 +238,12 @@ public class ReportAction
int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
recordCount += recordsConsumed;
long reportEndTime = System.currentTimeMillis();
LOG.info((countFromPreExecute != null
? String.format("Processed %,d of %,d records", recordCount, countFromPreExecute)
: String.format("Processed %,d records", recordCount))
+ String.format(" at end of report in %,d ms (%.2f records/second).", (reportEndTime - reportStartTime), 1000d * (recordCount / (.001d + (reportEndTime - reportStartTime)))));
//////////////////////////////////////////////////////////////////
// Critical: we must close the stream here as our final action //
//////////////////////////////////////////////////////////////////
@ -247,11 +266,40 @@ public class ReportAction
/*******************************************************************************
**
*******************************************************************************/
private List<QFieldMetaData> getFields(ReportInput reportInput)
{
QTableMetaData table = reportInput.getTable();
if(reportInput.getFieldNames() != null)
{
return (reportInput.getFieldNames().stream().map(table::getField).toList());
}
else
{
return (new ArrayList<>(table.getFields().values()));
}
}
/*******************************************************************************
**
*******************************************************************************/
private void verifyCountUnderMax(ReportInput reportInput, QBackendModuleInterface backendModule, ReportFormat reportFormat) throws QException
{
if(reportFormat.getMaxCols() != null)
{
List<QFieldMetaData> fields = getFields(reportInput);
if (fields.size() > reportFormat.getMaxCols())
{
throw (new QUserFacingException("The requested report would include more columns ("
+ String.format("%,d", fields.size()) + ") than the maximum allowed ("
+ String.format("%,d", reportFormat.getMaxCols()) + ") for the selected file format (" + reportFormat + ")."));
}
}
if(reportFormat.getMaxRows() != null)
{
if(reportInput.getLimit() == null || reportInput.getLimit() > reportFormat.getMaxRows())

20
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java
View File

@ -22,12 +22,10 @@
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
@ -38,7 +36,7 @@ public interface ReportStreamerInterface
/*******************************************************************************
** Called once, before any rows are available. Meant to write a header, for example.
*******************************************************************************/
void start(ReportInput reportInput) throws QReportingException;
void start(ReportInput reportInput, List<QFieldMetaData> fields) throws QReportingException;
/*******************************************************************************
** Called as records flow into the pipe.
@ -50,20 +48,4 @@ public interface ReportStreamerInterface
*******************************************************************************/
void finish() throws QReportingException;
/*******************************************************************************
** (Ideally, protected) method used within report streamer implementations, to
** map field names from reportInput into list of fieldMetaData.
*******************************************************************************/
default List<QFieldMetaData> setupFieldList(QTableMetaData table, ReportInput reportInput)
{
if(reportInput.getFieldNames() != null)
{
return (reportInput.getFieldNames().stream().map(table::getField).toList());
}
else
{
return (new ArrayList<>(table.getFields().values()));
}
}
}

1
src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java
View File

@ -77,6 +77,7 @@ public class DeleteAction
LOG.info("0 primaryKeys found. Returning with no-op");
DeleteOutput deleteOutput = new DeleteOutput();
deleteOutput.setRecordsWithErrors(new ArrayList<>());
deleteOutput.setDeletedRecordCount(0);
return (deleteOutput);
}
}

13
src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -246,14 +246,12 @@ public class QInstanceEnricher
.withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "Upload a CSV or XLSX file with the following columns: " + fieldsForHelpText));
// .withValue("text", "Upload a CSV or XLSX file with the following columns: " + fieldsForHelpText));
.withValue("text", "Upload a CSV file with the following columns: " + fieldsForHelpText));
QBackendStepMetaData receiveFileStep = new QBackendStepMetaData()
.withName("receiveFile")
.withCode(new QCodeReference(BulkInsertReceiveFileStep.class))
.withInputData(new QFunctionInputMetaData()
// todo - our upload file as a field? problem is, its type...
.withFieldList(List.of()))
.withOutputMetaData(new QFunctionOutputMetaData()
.withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
@ -268,9 +266,6 @@ public class QInstanceEnricher
QBackendStepMetaData storeStep = new QBackendStepMetaData()
.withName("storeRecords")
.withCode(new QCodeReference(BulkInsertStoreRecordsStep.class))
.withInputData(new QFunctionInputMetaData()
// todo - our upload file as a field? problem is, its type...
.withFieldList(List.of()))
.withOutputMetaData(new QFunctionOutputMetaData()
.withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
@ -342,15 +337,13 @@ public class QInstanceEnricher
QBackendStepMetaData storeStep = new QBackendStepMetaData()
.withName("storeRecords")
.withCode(new QCodeReference(BulkEditStoreRecordsStep.class))
.withInputData(new QFunctionInputMetaData()
// todo - our upload file as a field? problem is, its type...
.withFieldList(List.of()))
.withOutputMetaData(new QFunctionOutputMetaData()
.withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData()
.withName("results")
.withRecordListFields(new ArrayList<>(table.getFields().values()))
.withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below have been updated."));

2
src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/QUploadedFile.java
View File

@ -30,6 +30,8 @@ import java.io.Serializable;
*******************************************************************************/
public class QUploadedFile implements Serializable
{
public static final String DEFAULT_UPLOADED_FILE_FIELD_NAME = "uploadedFileKey";
private String filename;
private byte[] bytes;

25
src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
import java.io.Serializable;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
@ -204,7 +205,7 @@ public class RunBackendStepOutput extends AbstractActionOutput
*******************************************************************************/
public String getValueString(String fieldName)
{
return ((String) getValue(fieldName));
return (ValueUtils.getValueAsString(getValue(fieldName)));
}
@ -218,4 +219,26 @@ public class RunBackendStepOutput extends AbstractActionOutput
return (ValueUtils.getValueAsInteger(getValue(fieldName)));
}
/*******************************************************************************
** Getter for a single field's value
**
*******************************************************************************/
public Boolean getValueBoolean(String fieldName)
{
return (ValueUtils.getValueAsBoolean(getValue(fieldName)));
}
/*******************************************************************************
** Getter for a single field's value
**
*******************************************************************************/
public BigDecimal getValueBigDecimal(String fieldName)
{
return (ValueUtils.getValueAsBigDecimal(getValue(fieldName)));
}
}

18
src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java
View File

@ -37,11 +37,12 @@ import org.dhatim.fastexcel.Worksheet;
*******************************************************************************/
public enum ReportFormat
{
XLSX(Worksheet.MAX_ROWS, ExcelReportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
CSV(null, CsvReportStreamer::new, "text/csv");
XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelReportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
CSV(null, null, CsvReportStreamer::new, "text/csv");
private final Integer maxRows;
private final Integer maxCols;
private final String mimeType;
private final Supplier<? extends ReportStreamerInterface> streamerConstructor;
@ -51,9 +52,10 @@ public enum ReportFormat
/*******************************************************************************
**
*******************************************************************************/
ReportFormat(Integer maxRows, Supplier<? extends ReportStreamerInterface> streamerConstructor, String mimeType)
ReportFormat(Integer maxRows, Integer maxCols, Supplier<? extends ReportStreamerInterface> streamerConstructor, String mimeType)
{
this.maxRows = maxRows;
this.maxCols = maxCols;
this.mimeType = mimeType;
this.streamerConstructor = streamerConstructor;
}
@ -92,6 +94,16 @@ public enum ReportFormat
}
/*******************************************************************************
** Getter for maxCols
**
*******************************************************************************/
public Integer getMaxCols()
{
return maxCols;
}
/*******************************************************************************
** Getter for mimeType

12
src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java
View File

@ -58,8 +58,16 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
@Override
public void addRecord(QRecord record)
{
recordPipe.addRecord(record);
blockIfPipeIsTooFull();
if(!recordPipe.addRecord(record))
{
do
{
LOG.debug("Record pipe.add failed (due to full pipe). Blocking.");
SleepUtils.sleep(10, TimeUnit.MILLISECONDS);
}
while(!recordPipe.addRecord(record));
LOG.debug("Done blocking.");
}
}

3
src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java
View File

@ -42,6 +42,9 @@ public enum QFieldType
HTML,
PASSWORD,
BLOB;
///////////////////////////////////////////////////////////////////////
// keep these values in sync with QFieldType.ts in qqq-frontend-core //
///////////////////////////////////////////////////////////////////////

16
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java
View File

@ -24,7 +24,9 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import java.io.IOException;
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -33,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -43,6 +46,9 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
*******************************************************************************/
public class BulkDeleteStoreStep implements BackendStep
{
public static final String ERROR_COUNT = "errorCount";
/*******************************************************************************
**
@ -50,7 +56,7 @@ public class BulkDeleteStoreStep implements BackendStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records in database...");
runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records...");
runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal();
DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance());
@ -78,12 +84,16 @@ public class BulkDeleteStoreStep implements BackendStep
.toList();
deleteInput.setPrimaryKeys(primaryKeyList);
}
else
{
throw (new QException("Missing required inputs (queryFilterJSON or record list)"));
}
DeleteAction deleteAction = new DeleteAction();
DeleteOutput deleteOutput = deleteAction.execute(deleteInput);
// todo - something with the output!!
deleteOutput.getRecordsWithErrors();
List<QRecord> recordsWithErrors = Objects.requireNonNullElse(deleteOutput.getRecordsWithErrors(), Collections.emptyList());
runBackendStepOutput.addValue(ERROR_COUNT, recordsWithErrors.size());
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
}

26
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java
View File

@ -23,16 +23,11 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
@ -68,26 +63,7 @@ public class BulkEditReceiveValuesStep implements BackendStep
}
}
/////////////////////////////////////////////////////////////////////
// build the string to show the user what fields are being changed //
/////////////////////////////////////////////////////////////////////
List<String> valuesBeingUpdated = new ArrayList<>();
QTableMetaData table = runBackendStepInput.getTable();
for(String fieldName : enabledFields)
{
String label = table.getField(fieldName).getLabel();
Serializable value = runBackendStepInput.getValue(fieldName);
if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
{
valuesBeingUpdated.add(label + " will be set to: " + value);
}
else
{
valuesBeingUpdated.add(label + " will be cleared out.");
}
}
runBackendStepOutput.addValue(FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated));
BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "will be");
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
}

4
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java
View File

@ -69,7 +69,7 @@ public class BulkEditStoreRecordsStep implements BackendStep
}
}
runBackendStepInput.getAsyncJobCallback().updateStatus("Updating database...");
runBackendStepInput.getAsyncJobCallback().updateStatus("Storing updated records...");
runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal();
UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance());
@ -83,6 +83,8 @@ public class BulkEditStoreRecordsStep implements BackendStep
UpdateOutput updateOutput = updateAction.execute(updateInput);
runBackendStepOutput.setRecords(updateOutput.getRecords());
BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "was");
}
}

68
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java Normal file
View File

@ -0,0 +1,68 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** Utility methods used for Bulk Edit steps
*******************************************************************************/
public class BulkEditUtils
{
/*******************************************************************************
**
*******************************************************************************/
public static void setFieldValuesBeingUpdated(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, String[] enabledFields, String verb)
{
/////////////////////////////////////////////////////////////////////
// build the string to show the user what fields are being changed //
/////////////////////////////////////////////////////////////////////
List<String> valuesBeingUpdated = new ArrayList<>();
QTableMetaData table = runBackendStepInput.getTable();
for(String fieldName : enabledFields)
{
String label = table.getField(fieldName).getLabel();
Serializable value = runBackendStepInput.getValue(fieldName);
if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
{
valuesBeingUpdated.add(label + " " + verb + " set to: " + value);
}
else
{
valuesBeingUpdated.add(label + " " + verb + " cleared out");
}
}
runBackendStepOutput.addValue(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated));
}
}

24
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java
View File

@ -23,10 +23,12 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
@ -47,28 +49,36 @@ public class BulkInsertUtils
*******************************************************************************/
static List<QRecord> getQRecordsFromFile(RunBackendStepInput runBackendStepInput) throws QException
{
AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue("uploadedFileKey");
AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME);
Optional<QUploadedFile> optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey);
if(optionalUploadedFile.isEmpty())
{
throw (new QException("Could not find uploaded file"));
}
byte[] bytes = optionalUploadedFile.get().getBytes();
byte[] bytes = optionalUploadedFile.get().getBytes();
String fileName = optionalUploadedFile.get().getFilename();
/////////////////////////////////////////////////////
// let the user specify field labels instead names //
/////////////////////////////////////////////////////
QTableMetaData table = runBackendStepInput.getTable();
QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping();
QTableMetaData table = runBackendStepInput.getTable();
String tableName = runBackendStepInput.getTableName();
QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping();
for(Map.Entry<String, QFieldMetaData> entry : table.getFields().entrySet())
{
mapping.addMapping(entry.getKey(), entry.getValue().getLabel());
}
// todo - sniff out file type...
String tableName = runBackendStepInput.getTableName();
List<QRecord> qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping);
List<QRecord> qRecords;
if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv"))
{
qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping);
}
else
{
throw (new QUserFacingException("Unsupported file type."));
}
////////////////////////////////////////////////
// remove values from any non-editable fields //

1
src/main/java/com/kingsrook/qqq/backend/core/state/StateType.java
View File

@ -32,6 +32,5 @@ public enum StateType
{
PROCESS_STATUS,
ASYNC_JOB_STATUS,
ASYNC_JOB_RESULT,
UPLOADED_FILE
}

66
src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java
View File

@ -34,7 +34,9 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.apache.commons.io.FileUtils;
@ -70,6 +72,27 @@ class ReportActionTest
/*******************************************************************************
** This test runs for more records, to stress more of the pipe-filling and
** other bits of the ReportAction.
*******************************************************************************/
@Test
public void testBigger() throws Exception
{
// int recordCount = 2_000_000; // to really stress locally, use this.
int recordCount = 200_000;
String filename = "/tmp/ReportActionTest.csv";
runReport(recordCount, filename, ReportFormat.CSV, false);
File file = new File(filename);
List<String> fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name());
assertEquals(recordCount + 1, fileLines.size());
assertTrue(file.delete());
}
/*******************************************************************************
**
*******************************************************************************/
@ -142,12 +165,49 @@ class ReportActionTest
ReportInput reportInput = new ReportInput(TestUtils.defineInstance(), TestUtils.getMockSession());
reportInput.setTableName("person");
////////////////////////////////////////////////////////////////
// use xlsx, which has a max-rows limit, to verify that code. //
////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// use xlsx, which has a max-rows limit, to verify that code runs, but doesn't throw when there aren't too many rows //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
reportInput.setReportFormat(ReportFormat.XLSX);
new ReportAction().preExecute(reportInput);
////////////////////////////////////////////////////////////////////////////
// nothing to assert - but if preExecute throws, then the test will fail. //
////////////////////////////////////////////////////////////////////////////
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testTooManyColumns() throws QException
{
QTableMetaData wideTable = new QTableMetaData()
.withName("wide")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME);
for(int i = 0; i < ReportFormat.XLSX.getMaxCols() + 1; i++)
{
wideTable.addField(new QFieldMetaData("field" + i, QFieldType.STRING));
}
QInstance qInstance = TestUtils.defineInstance();
qInstance.addTable(wideTable);
ReportInput reportInput = new ReportInput(qInstance, TestUtils.getMockSession());
reportInput.setTableName("wide");
////////////////////////////////////////////////////////////////
// use xlsx, which has a max-cols limit, to verify that code. //
////////////////////////////////////////////////////////////////
reportInput.setReportFormat(ReportFormat.XLSX);
assertThrows(QUserFacingException.class, () ->
{
new ReportAction().preExecute(reportInput);
});
}
}

3
src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java
View File

@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -50,6 +51,7 @@ class BulkDeleteStoreStepTest
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkDeleteStoreStep().run(stepInput, stepOutput);
assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
}
@ -66,6 +68,7 @@ class BulkDeleteStoreStepTest
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkDeleteStoreStep().run(stepInput, stepOutput);
assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
}
}

35
src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@ -63,7 +64,7 @@ class BulkInsertReceiveFileStepTest
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue("uploadedFileKey", key);
stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkInsertReceiveFileStep().run(stepInput, stepOutput);
@ -78,4 +79,36 @@ class BulkInsertReceiveFileStepTest
assertEquals(2, stepOutput.getValueInteger("noOfFileRows"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBadFileType() throws QException
{
////////////////////////////////////////////////////////////////
// create an uploaded file, similar to how an http server may //
////////////////////////////////////////////////////////////////
QUploadedFile qUploadedFile = new QUploadedFile();
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); // todo - this is NOT excel content...
qUploadedFile.setFilename("test.xslx");
UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
TempFileStateProvider.getInstance().put(key, qUploadedFile);
////////////////////////////
// setup and run the step //
////////////////////////////
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
assertThrows(QUserFacingException.class, () ->
{
new BulkInsertReceiveFileStep().run(stepInput, stepOutput);
});
}
}

2
src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java
View File

@ -64,7 +64,7 @@ class BulkInsertStoreRecordsStepTest
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue("uploadedFileKey", key);
stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkInsertStoreRecordsStep().run(stepInput, stepOutput);

8
src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -335,7 +335,7 @@ public class TestUtils
public static String getPersonCsvHeader()
{
return ("""
"id","createDate","modifyDate","firstName","lastName","birthDate","email"\r
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
""");
}
@ -347,7 +347,7 @@ public class TestUtils
public static String getPersonCsvHeaderUsingLabels()
{
return ("""
"Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"\r
"Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
""");
}
@ -359,7 +359,7 @@ public class TestUtils
public static String getPersonCsvRow1()
{
return ("""
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"\r
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
""");
}
@ -371,7 +371,7 @@ public class TestUtils
public static String getPersonCsvRow2()
{
return ("""
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"\r
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
""");
}