diff --git a/.circleci/config.yml b/.circleci/config.yml index fe4c371c..658f75d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,8 @@ commands: name: Run Maven command: | mvn -s .circleci/mvn-settings.xml << parameters.maven_subcommand >> + - store_artifacts: + path: target/site/jacoco - run: name: Save test results command: | diff --git a/.gitignore b/.gitignore index 39736a21..b65cb041 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ target/ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* +.DS_Store diff --git a/pom.xml b/pom.xml index 077ba675..4e88d244 100644 --- a/pom.xml +++ b/pom.xml @@ -79,11 +79,18 @@ 3.23.1 test + com.github.hervian safety-mirror 4.0.1 + + + org.dhatim + fastexcel + 0.12.15 + diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index 4e3acc3f..94106b7f 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider; import com.kingsrook.qqq.backend.core.state.StateProviderInterface; import com.kingsrook.qqq.backend.core.state.StateType; import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -49,10 +50,21 @@ public class AsyncJobManager /******************************************************************************* - ** Run a job - if it finishes within the specified timeout, get its results, + ** Start a job - if it finishes within the specified timeout, get its results, ** else, get back an exception with the job id. *******************************************************************************/ public T startJob(long timeout, TimeUnit timeUnit, AsyncJob asyncJob) throws JobGoingAsyncException, QException + { + return (startJob("Anonymous", timeout, timeUnit, asyncJob)); + } + + + + /******************************************************************************* + ** Start a job - if it finishes within the specified timeout, get its results, + ** else, get back an exception with the job id. + *******************************************************************************/ + public T startJob(String jobName, long timeout, TimeUnit timeUnit, AsyncJob asyncJob) throws JobGoingAsyncException, QException { UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS); AsyncJobStatus asyncJobStatus = new AsyncJobStatus(); @@ -63,22 +75,7 @@ public class AsyncJobManager { CompletableFuture future = CompletableFuture.supplyAsync(() -> { - try - { - LOG.info("Starting job " + uuidAndTypeStateKey.getUuid()); - T result = asyncJob.run(new AsyncJobCallback(uuidAndTypeStateKey.getUuid(), asyncJobStatus)); - asyncJobStatus.setState(AsyncJobState.COMPLETE); - getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); - LOG.info("Completed job " + uuidAndTypeStateKey.getUuid()); - return (result); - } - catch(Exception e) - { - asyncJobStatus.setState(AsyncJobState.ERROR); - asyncJobStatus.setCaughtException(e); - getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); - throw (new CompletionException(e)); - } + return (runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus)); }); T result = future.get(timeout, timeUnit); @@ -97,6 +94,66 @@ public class AsyncJobManager + /******************************************************************************* + ** Start a job, and always, just get back the job UUID. + *******************************************************************************/ + public String startJob(AsyncJob asyncJob) + { + return (startJob("Anonymous", asyncJob)); + } + + + + /******************************************************************************* + ** Start a job, and always, just get back the job UUID. + *******************************************************************************/ + public String startJob(String jobName, AsyncJob asyncJob) + { + UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS); + AsyncJobStatus asyncJobStatus = new AsyncJobStatus(); + asyncJobStatus.setState(AsyncJobState.RUNNING); + getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); + + // todo - refactor, share + CompletableFuture.supplyAsync(() -> runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus)); + + return (uuidAndTypeStateKey.getUuid().toString()); + } + + + + /******************************************************************************* + ** run the job. + *******************************************************************************/ + private T runAsyncJob(String jobName, AsyncJob asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus) + { + String originalThreadName = Thread.currentThread().getName(); + Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8)); + try + { + LOG.info("Starting job " + uuidAndTypeStateKey.getUuid()); + T result = asyncJob.run(new AsyncJobCallback(uuidAndTypeStateKey.getUuid(), asyncJobStatus)); + asyncJobStatus.setState(AsyncJobState.COMPLETE); + getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); + LOG.info("Completed job " + uuidAndTypeStateKey.getUuid()); + return (result); + } + catch(Exception e) + { + asyncJobStatus.setState(AsyncJobState.ERROR); + asyncJobStatus.setCaughtException(e); + getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); + LOG.warn("Job " + uuidAndTypeStateKey.getUuid() + " ended with an exception: ", e); + throw (new CompletionException(e)); + } + finally + { + Thread.currentThread().setName(originalThreadName); + } + } + + + /******************************************************************************* ** Get the status of the job identified by the given UUID. ** @@ -122,4 +179,31 @@ public class AsyncJobManager // return TempFileStateProvider.getInstance(); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public AsyncJobStatus waitForJob(String jobUUID) throws QException + { + AsyncJobState asyncJobState = AsyncJobState.RUNNING; + AsyncJobStatus asyncJobStatus = null; + while(asyncJobState.equals(AsyncJobState.RUNNING)) + { + LOG.info("Sleeping, waiting on job [" + jobUUID + "]"); + SleepUtils.sleep(100, TimeUnit.MILLISECONDS); + + Optional optionalAsyncJobStatus = getJobStatus(jobUUID); + if(optionalAsyncJobStatus.isEmpty()) + { + // todo - ... maybe some version of try-again? + throw (new QException("Could not get status of report query job [" + jobUUID + "]")); + } + asyncJobStatus = optionalAsyncJobStatus.get(); + asyncJobState = asyncJobStatus.getState(); + } + + return (asyncJobStatus); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java new file mode 100644 index 00000000..217e7d4c --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java @@ -0,0 +1,143 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import com.kingsrook.qqq.backend.core.adapters.QRecordToCsvAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** CSV report format implementation + *******************************************************************************/ +public class CsvReportStreamer implements ReportStreamerInterface +{ + private static final Logger LOG = LogManager.getLogger(CsvReportStreamer.class); + + private final QRecordToCsvAdapter qRecordToCsvAdapter; + + private ReportInput reportInput; + private QTableMetaData table; + private List fields; + private OutputStream outputStream; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public CsvReportStreamer() + { + qRecordToCsvAdapter = new QRecordToCsvAdapter(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void start(ReportInput reportInput) throws QReportingException + { + this.reportInput = reportInput; + table = reportInput.getTable(); + outputStream = this.reportInput.getReportOutputStream(); + + fields = setupFieldList(table, reportInput); + + writeReportHeaderRow(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeReportHeaderRow() throws QReportingException + { + try + { + int col = 0; + for(QFieldMetaData column : fields) + { + if(col++ > 0) + { + outputStream.write(','); + } + outputStream.write(('"' + column.getLabel() + '"').getBytes(StandardCharsets.UTF_8)); + } + outputStream.write('\n'); + } + catch(Exception e) + { + throw (new QReportingException("Error starting CSV report")); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int takeRecordsFromPipe(RecordPipe recordPipe) throws QReportingException + { + List qRecords = recordPipe.consumeAvailableRecords(); + LOG.info("Consuming [" + qRecords.size() + "] records from the pipe"); + + try + { + for(QRecord qRecord : qRecords) + { + String csv = qRecordToCsvAdapter.recordToCsv(table, qRecord, fields); + outputStream.write(csv.getBytes(StandardCharsets.UTF_8)); + outputStream.flush(); // todo - less often? + } + return (qRecords.size()); + } + catch(Exception e) + { + throw (new QReportingException("Error writing CSV report", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void finish() + { + + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java new file mode 100644 index 00000000..ec1c95bf --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java @@ -0,0 +1,213 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.io.OutputStream; +import java.io.Serializable; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.Date; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dhatim.fastexcel.Workbook; +import org.dhatim.fastexcel.Worksheet; + + +/******************************************************************************* + ** Excel report format implementation + *******************************************************************************/ +public class ExcelReportStreamer implements ReportStreamerInterface +{ + private static final Logger LOG = LogManager.getLogger(ExcelReportStreamer.class); + + private ReportInput reportInput; + private QTableMetaData table; + private List fields; + private OutputStream outputStream; + + private Workbook workbook; + private Worksheet worksheet; + private int row = 1; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ExcelReportStreamer() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void start(ReportInput reportInput) throws QReportingException + { + this.reportInput = reportInput; + table = reportInput.getTable(); + outputStream = this.reportInput.getReportOutputStream(); + + workbook = new Workbook(outputStream, "QQQ", null); + worksheet = workbook.newWorksheet("Sheet 1"); + + fields = setupFieldList(table, reportInput); + + writeReportHeaderRow(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeReportHeaderRow() throws QReportingException + { + try + { + int col = 0; + for(QFieldMetaData column : fields) + { + worksheet.value(0, col, column.getLabel()); + col++; + } + } + catch(Exception e) + { + throw (new QReportingException("Error starting Excel report")); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int takeRecordsFromPipe(RecordPipe recordPipe) throws QReportingException + { + List qRecords = recordPipe.consumeAvailableRecords(); + LOG.info("Consuming [" + qRecords.size() + "] records from the pipe"); + + try + { + for(QRecord qRecord : qRecords) + { + int col = 0; + for(QFieldMetaData column : fields) + { + Serializable value = qRecord.getValue(column.getName()); + if(value != null) + { + if(value instanceof String s) + { + worksheet.value(row, col, s); + } + else if(value instanceof Number n) + { + worksheet.value(row, col, n); + } + else if(value instanceof Boolean b) + { + worksheet.value(row, col, b); + } + else if(value instanceof Date d) + { + worksheet.value(row, col, d); + worksheet.style(row, col).format("yyyy-MM-dd").set(); + } + else if(value instanceof LocalDate d) + { + worksheet.value(row, col, d); + worksheet.style(row, col).format("yyyy-MM-dd").set(); + } + else if(value instanceof LocalDateTime d) + { + worksheet.value(row, col, d); + worksheet.style(row, col).format("yyyy-MM-dd H:mm:ss").set(); + } + else if(value instanceof ZonedDateTime d) + { + worksheet.value(row, col, d); + worksheet.style(row, col).format("yyyy-MM-dd H:mm:ss").set(); + } + else + { + worksheet.value(row, col, ValueUtils.getValueAsString(value)); + } + } + col++; + } + + row++; + worksheet.flush(); // todo? not at all? or just sometimes? + } + } + catch(Exception e) + { + try + { + workbook.finish(); + outputStream.close(); + } + finally + { + throw (new QReportingException("Error generating Excel report", e)); + } + } + + return (qRecords.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void finish() throws QReportingException + { + try + { + if(workbook != null) + { + workbook.finish(); + } + } + catch(Exception e) + { + throw (new QReportingException("Error finishing Excel report", e)); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java new file mode 100644 index 00000000..b03ba662 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Object to connect a producer of records with a consumer. + ** Best for those to be on different threads, to avoid deadlock. + *******************************************************************************/ +public class RecordPipe +{ + private Queue queue = new ArrayDeque<>(); + + + + /******************************************************************************* + ** Add a record to the pipe + *******************************************************************************/ + public void addRecord(QRecord record) + { + queue.add(record); + } + + + + /******************************************************************************* + ** Add a list of records to the pipe + *******************************************************************************/ + public void addRecords(List records) + { + queue.addAll(records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public List consumeAvailableRecords() + { + List rs = new ArrayList<>(); + + while(true) + { + QRecord record = queue.poll(); + if(record == null) + { + break; + } + rs.add(record); + } + + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public int countAvailableRecords() + { + return (queue.size()); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java new file mode 100644 index 00000000..fcdd0786 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java @@ -0,0 +1,265 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Action to generate a report. + ** + ** At this time (future may change?), this action starts a new thread to run + ** the query in the backend module. As records are produced by the query, + ** they are put into a RecordPipe, which the ReportStreamer pulls from, to write + ** to the report output stream. This action will block until the query job + ** is complete, and the final records have been consumed from the pipe, at which + ** time the report outputStream can be closed. + ** + *******************************************************************************/ +public class ReportAction +{ + private static final Logger LOG = LogManager.getLogger(ReportAction.class); + + private boolean preExecuteRan = false; + + + + /******************************************************************************* + ** Validation logic, that will run before the action is executed -- ideally, if + ** a caller is going to run the execution in a thread, they'd call this method + ** first, in their thread, to catch any validation errors before they start + ** the thread (which they may abandon). + *******************************************************************************/ + public void preExecute(ReportInput reportInput) throws QException + { + ActionHelper.validateSession(reportInput); + + QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); + QBackendModuleInterface backendModule = qBackendModuleDispatcher.getQBackendModule(reportInput.getBackend()); + + /////////////////////////////////// + // verify field names (if given) // + /////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(reportInput.getFieldNames())) + { + QTableMetaData table = reportInput.getTable(); + List badFieldNames = new ArrayList<>(); + for(String fieldName : reportInput.getFieldNames()) + { + try + { + table.getField(fieldName); + } + catch(IllegalArgumentException iae) + { + badFieldNames.add(fieldName); + } + } + + if(!badFieldNames.isEmpty()) + { + throw (new QUserFacingException(badFieldNames.size() == 1 + ? ("Field name " + badFieldNames.get(0) + " was not found on the " + table.getLabel() + " table.") + : ("Fields names " + StringUtils.joinWithCommasAndAnd(badFieldNames) + " were not found on the " + table.getLabel() + " table."))); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // check if this report format has a max-rows limit -- if so, do a count to verify we're under the limit // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + ReportFormat reportFormat = reportInput.getReportFormat(); + verifyCountUnderMax(reportInput, backendModule, reportFormat); + + preExecuteRan = true; + } + + + + /******************************************************************************* + ** Run the report. + *******************************************************************************/ + public ReportOutput execute(ReportInput reportInput) throws QException + { + if(!preExecuteRan) + { + ///////////////////////////////////// + // ensure that pre-execute has ran // + ///////////////////////////////////// + preExecute(reportInput); + } + + QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); + QBackendModuleInterface backendModule = qBackendModuleDispatcher.getQBackendModule(reportInput.getBackend()); + QTableMetaData table = reportInput.getTable(); + + ////////////////////////// + // set up a query input // + ////////////////////////// + QueryInterface queryInterface = backendModule.getQueryInterface(); + QueryInput queryInput = new QueryInput(reportInput.getInstance()); + queryInput.setSession(reportInput.getSession()); + queryInput.setTableName(reportInput.getTableName()); + queryInput.setFilter(reportInput.getQueryFilter()); + queryInput.setLimit(reportInput.getLimit()); + + ///////////////////////////////////////////////////////////////// + // tell this query that it needs to put its output into a pipe // + ///////////////////////////////////////////////////////////////// + RecordPipe recordPipe = new RecordPipe(); + queryInput.setRecordPipe(recordPipe); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up a report streamer, which will read rows from the pipe, and write formatted report rows to the output stream // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ReportFormat reportFormat = reportInput.getReportFormat(); + ReportStreamerInterface reportStreamer = reportFormat.newReportStreamer(); + reportStreamer.start(reportInput); + + ////////////////////////////////////////// + // run the query action as an async job // + ////////////////////////////////////////// + AsyncJobManager asyncJobManager = new AsyncJobManager(); + String jobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput))); + LOG.info("Started query job [" + jobUUID + "] for report"); + + AsyncJobState asyncJobState = AsyncJobState.RUNNING; + AsyncJobStatus asyncJobStatus = null; + int nextSleepMillis = 10; + long recordCount = 0; + while(asyncJobState.equals(AsyncJobState.RUNNING)) + { + int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe); + recordCount += recordsConsumed; + if(recordsConsumed == 0) + { + ////////////////////////////////////////////////////////////////////////////////// + // do we need to sleep to let the producer work? // + // todo - smarter sleep? like an exponential back-off? and eventually a fail? // + ////////////////////////////////////////////////////////////////////////////////// + LOG.info("Read 0 records from pipe, sleeping to give producer a chance to work"); + SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); + while(recordPipe.countAvailableRecords() == 0) + { + nextSleepMillis *= 2; + LOG.info("Still no records in the pipe, so sleeping more [" + nextSleepMillis + "]ms"); + SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); + } + } + + nextSleepMillis = 10; + + Optional optionalAsyncJobStatus = asyncJobManager.getJobStatus(jobUUID); + if(optionalAsyncJobStatus.isEmpty()) + { + ///////////////////////////////////////////////// + // todo - ... maybe some version of try-again? // + ///////////////////////////////////////////////// + throw (new QException("Could not get status of report query job [" + jobUUID + "]")); + } + asyncJobStatus = optionalAsyncJobStatus.get(); + asyncJobState = asyncJobStatus.getState(); + } + + LOG.info("Query job [" + jobUUID + "] for report completed with status: " + asyncJobStatus); + + /////////////////////////////////////////////////// + // send the final records to the report streamer // + /////////////////////////////////////////////////// + int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe); + recordCount += recordsConsumed; + + ////////////////////////////////////////////////////////////////// + // Critical: we must close the stream here as our final action // + ////////////////////////////////////////////////////////////////// + reportStreamer.finish(); + + try + { + reportInput.getReportOutputStream().close(); + } + catch(Exception e) + { + throw (new QReportingException("Error completing report", e)); + } + + ReportOutput reportOutput = new ReportOutput(); + reportOutput.setRecordCount(recordCount); + + return (reportOutput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void verifyCountUnderMax(ReportInput reportInput, QBackendModuleInterface backendModule, ReportFormat reportFormat) throws QException + { + if(reportFormat.getMaxRows() != null) + { + if(reportInput.getLimit() == null || reportInput.getLimit() > reportFormat.getMaxRows()) + { + CountInterface countInterface = backendModule.getCountInterface(); + CountInput countInput = new CountInput(reportInput.getInstance()); + countInput.setSession(reportInput.getSession()); + countInput.setTableName(reportInput.getTableName()); + countInput.setFilter(reportInput.getQueryFilter()); + CountOutput countOutput = countInterface.execute(countInput); + Integer count = countOutput.getCount(); + if(count > reportFormat.getMaxRows()) + { + throw (new QUserFacingException("The requested report would include more rows (" + + String.format("%,d", count) + ") than the maximum allowed (" + + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ").")); + } + } + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java new file mode 100644 index 00000000..fe65fce3 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Interface for various report formats to implement. + *******************************************************************************/ +public interface ReportStreamerInterface +{ + /******************************************************************************* + ** Called once, before any rows are available. Meant to write a header, for example. + *******************************************************************************/ + void start(ReportInput reportInput) throws QReportingException; + + /******************************************************************************* + ** Called as records flow into the pipe. + ******************************************************************************/ + int takeRecordsFromPipe(RecordPipe recordPipe) throws QReportingException; + + /******************************************************************************* + ** Called once, after all rows are available. Meant to write a footer, or close resources, for example. + *******************************************************************************/ + void finish() throws QReportingException; + + /******************************************************************************* + ** (Ideally, protected) method used within report streamer implementations, to + ** map field names from reportInput into list of fieldMetaData. + *******************************************************************************/ + default List setupFieldList(QTableMetaData table, ReportInput reportInput) + { + if(reportInput.getFieldNames() != null) + { + return (reportInput.getFieldNames().stream().map(table::getField).toList()); + } + else + { + return (new ArrayList<>(table.getFields().values())); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java b/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java new file mode 100644 index 00000000..7319c779 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.adapters; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Class to convert QRecords to CSV Strings. + *******************************************************************************/ +public class QRecordToCsvAdapter +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public String recordToCsv(QTableMetaData table, QRecord record) + { + return (recordToCsv(table, record, new ArrayList<>(table.getFields().values()))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public String recordToCsv(QTableMetaData table, QRecord record, List fields) + { + StringBuilder rs = new StringBuilder(); + int fieldNo = 0; + + for(QFieldMetaData field : fields) + { + if(fieldNo++ > 0) + { + rs.append(','); + } + rs.append('"'); + Serializable value = record.getValue(field.getName()); + String valueAsString = ValueUtils.getValueAsString(value); + if(StringUtils.hasContent(valueAsString)) + { + rs.append(sanitize(valueAsString)); + } + rs.append('"'); + } + rs.append('\n'); + return (rs.toString()); + } + + + + /******************************************************************************* + ** todo - kinda weak... can we find this in a CSV lib?? + *******************************************************************************/ + private String sanitize(String value) + { + return (value.replaceAll("\"", "\"\"").replaceAll("\n", " ")); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QReportingException.java b/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QReportingException.java new file mode 100644 index 00000000..e24b09a9 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QReportingException.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.exceptions; + + +/******************************************************************************* + * Exception thrown while generating reports + * + *******************************************************************************/ +public class QReportingException extends QException +{ + + /******************************************************************************* + ** Constructor of message + ** + *******************************************************************************/ + public QReportingException(String message) + { + super(message); + } + + + + /******************************************************************************* + ** Constructor of message & cause + ** + *******************************************************************************/ + public QReportingException(String message, Throwable cause) + { + super(message, cause); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java new file mode 100644 index 00000000..fd874601 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java @@ -0,0 +1,114 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import java.util.Locale; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.reporting.CsvReportStreamer; +import com.kingsrook.qqq.backend.core.actions.reporting.ExcelReportStreamer; +import com.kingsrook.qqq.backend.core.actions.reporting.ReportStreamerInterface; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.dhatim.fastexcel.Worksheet; + + +/******************************************************************************* + ** QQQ Report/export file formats + *******************************************************************************/ +public enum ReportFormat +{ + XLSX(Worksheet.MAX_ROWS, ExcelReportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), + CSV(null, CsvReportStreamer::new, "text/csv"); + + + private final Integer maxRows; + private final String mimeType; + + private final Supplier streamerConstructor; + + + + /******************************************************************************* + ** + *******************************************************************************/ + ReportFormat(Integer maxRows, Supplier streamerConstructor, String mimeType) + { + this.maxRows = maxRows; + this.mimeType = mimeType; + this.streamerConstructor = streamerConstructor; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ReportFormat fromString(String format) throws QUserFacingException + { + if(!StringUtils.hasContent(format)) + { + throw (new QUserFacingException("Report format was not specified.")); + } + + try + { + return (ReportFormat.valueOf(format.toUpperCase(Locale.ROOT))); + } + catch(IllegalArgumentException iae) + { + throw (new QUserFacingException("Unsupported report format: " + format + ".")); + } + } + + + + /******************************************************************************* + ** Getter for maxRows + ** + *******************************************************************************/ + public Integer getMaxRows() + { + return maxRows; + } + + + + /******************************************************************************* + ** Getter for mimeType + ** + *******************************************************************************/ + public String getMimeType() + { + return mimeType; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ReportStreamerInterface newReportStreamer() + { + return (streamerConstructor.get()); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java new file mode 100644 index 00000000..211799d2 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java @@ -0,0 +1,207 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import java.io.OutputStream; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** Input for a Report action + *******************************************************************************/ +public class ReportInput extends AbstractTableActionInput +{ + private QQueryFilter queryFilter; + private Integer limit; + private List fieldNames; + + private String filename; + private ReportFormat reportFormat; + private OutputStream reportOutputStream; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ReportInput() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ReportInput(QInstance instance) + { + super(instance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ReportInput(QInstance instance, QSession session) + { + super(instance); + setSession(session); + } + + + + /******************************************************************************* + ** Getter for queryFilter + ** + *******************************************************************************/ + public QQueryFilter getQueryFilter() + { + return queryFilter; + } + + + + /******************************************************************************* + ** Setter for queryFilter + ** + *******************************************************************************/ + public void setQueryFilter(QQueryFilter queryFilter) + { + this.queryFilter = queryFilter; + } + + + + /******************************************************************************* + ** Getter for limit + ** + *******************************************************************************/ + public Integer getLimit() + { + return limit; + } + + + + /******************************************************************************* + ** Setter for limit + ** + *******************************************************************************/ + public void setLimit(Integer limit) + { + this.limit = limit; + } + + + + /******************************************************************************* + ** Getter for fieldNames + ** + *******************************************************************************/ + public List getFieldNames() + { + return fieldNames; + } + + + + /******************************************************************************* + ** Setter for fieldNames + ** + *******************************************************************************/ + public void setFieldNames(List fieldNames) + { + this.fieldNames = fieldNames; + } + + + + /******************************************************************************* + ** Getter for filename + ** + *******************************************************************************/ + public String getFilename() + { + return filename; + } + + + + /******************************************************************************* + ** Setter for filename + ** + *******************************************************************************/ + public void setFilename(String filename) + { + this.filename = filename; + } + + + + /******************************************************************************* + ** Getter for reportFormat + ** + *******************************************************************************/ + public ReportFormat getReportFormat() + { + return reportFormat; + } + + + + /******************************************************************************* + ** Setter for reportFormat + ** + *******************************************************************************/ + public void setReportFormat(ReportFormat reportFormat) + { + this.reportFormat = reportFormat; + } + + + + /******************************************************************************* + ** Getter for reportOutputStream + ** + *******************************************************************************/ + public OutputStream getReportOutputStream() + { + return reportOutputStream; + } + + + + /******************************************************************************* + ** Setter for reportOutputStream + ** + *******************************************************************************/ + public void setReportOutputStream(OutputStream reportOutputStream) + { + this.reportOutputStream = reportOutputStream; + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java new file mode 100644 index 00000000..01e3e0de --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java @@ -0,0 +1,56 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import java.io.Serializable; + + +/******************************************************************************* + ** Output for a Report action + *******************************************************************************/ +public class ReportOutput implements Serializable +{ + public long recordCount; + + + + /******************************************************************************* + ** Getter for recordCount + ** + *******************************************************************************/ + public long getRecordCount() + { + return recordCount; + } + + + + /******************************************************************************* + ** Setter for recordCount + ** + *******************************************************************************/ + public void setRecordCount(long recordCount) + { + this.recordCount = recordCount; + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java index fddf2d14..febe1e9e 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -36,6 +37,8 @@ public class QueryInput extends AbstractTableActionInput private Integer skip; private Integer limit; + private RecordPipe recordPipe; + /******************************************************************************* @@ -120,4 +123,27 @@ public class QueryInput extends AbstractTableActionInput { this.limit = limit; } + + + + /******************************************************************************* + ** Getter for recordPipe + ** + *******************************************************************************/ + public RecordPipe getRecordPipe() + { + return recordPipe; + } + + + + /******************************************************************************* + ** Setter for recordPipe + ** + *******************************************************************************/ + public void setRecordPipe(RecordPipe recordPipe) + { + this.recordPipe = recordPipe; + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java index 9abb787d..a9e19342 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; +import java.io.Serializable; import java.util.List; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -31,9 +32,52 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** Output for a query action ** *******************************************************************************/ -public class QueryOutput extends AbstractActionOutput +public class QueryOutput extends AbstractActionOutput implements Serializable { - private List records; + private QueryOutputStorageInterface storage; + + + + /******************************************************************************* + ** Construct a new query output, based on a query input (which will drive some + ** of how our output is structured... e.g., if we pipe the output) + *******************************************************************************/ + public QueryOutput(QueryInput queryInput) + { + if(queryInput.getRecordPipe() != null) + { + storage = new QueryOutputRecordPipe(queryInput.getRecordPipe()); + } + else + { + storage = new QueryOutputList(); + } + } + + + + /******************************************************************************* + ** Add a record to this output. Note - we often don't care, in such a method, + ** whether the record is "completed" or not (e.g., all of its values have been + ** populated) - but - note in here - that this records MAY be going into a pipe + ** that could be read asynchronously, at any time, by another thread - SO - only + ** completely populated records should be passed into this method. + *******************************************************************************/ + public void addRecord(QRecord record) + { + storage.addRecord(record); + } + + + + /******************************************************************************* + ** add a list of records to this output + *******************************************************************************/ + public void addRecords(List records) + { + storage.addRecords(records); + } + /******************************************************************************* @@ -41,16 +85,7 @@ public class QueryOutput extends AbstractActionOutput *******************************************************************************/ public List getRecords() { - return records; + return storage.getRecords(); } - - - /******************************************************************************* - ** - *******************************************************************************/ - public void setRecords(List records) - { - this.records = records; - } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputList.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputList.java new file mode 100644 index 00000000..9d7ee0f4 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputList.java @@ -0,0 +1,80 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Kinda the standard way that a QueryOutput would store its records - in a + ** simple list. + *******************************************************************************/ +class QueryOutputList implements QueryOutputStorageInterface +{ + private List records = new ArrayList<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryOutputList() + { + } + + + + /******************************************************************************* + ** add a record to this output + *******************************************************************************/ + @Override + public void addRecord(QRecord record) + { + records.add(record); + } + + + + /******************************************************************************* + ** add a list of records to this output + *******************************************************************************/ + @Override + public void addRecords(List records) + { + this.records.addAll(records); + } + + + + /******************************************************************************* + ** Get all stored records + *******************************************************************************/ + @Override + public List getRecords() + { + return (records); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java new file mode 100644 index 00000000..52776550 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java @@ -0,0 +1,107 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query; + + +import java.util.List; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Query output that uses a RecordPipe + *******************************************************************************/ +class QueryOutputRecordPipe implements QueryOutputStorageInterface +{ + private static final Logger LOG = LogManager.getLogger(QueryOutputRecordPipe.class); + + private RecordPipe recordPipe; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryOutputRecordPipe(RecordPipe recordPipe) + { + this.recordPipe = recordPipe; + } + + + + /******************************************************************************* + ** add a record to this output + *******************************************************************************/ + @Override + public void addRecord(QRecord record) + { + recordPipe.addRecord(record); + blockIfPipeIsTooFull(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void blockIfPipeIsTooFull() + { + if(recordPipe.countAvailableRecords() >= 100_000) + { + LOG.info("Record pipe is kinda full. Blocking for a bit"); + do + { + SleepUtils.sleep(10, TimeUnit.MILLISECONDS); + } + while(recordPipe.countAvailableRecords() >= 10_000); + LOG.info("Done blocking."); + } + } + + + + /******************************************************************************* + ** add a list of records to this output + *******************************************************************************/ + @Override + public void addRecords(List records) + { + recordPipe.addRecords(records); + blockIfPipeIsTooFull(); + } + + + + /******************************************************************************* + ** Get all stored records + *******************************************************************************/ + @Override + public List getRecords() + { + throw (new IllegalStateException("getRecords may not be called on a piped query output")); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputStorageInterface.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputStorageInterface.java new file mode 100644 index 00000000..95046765 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputStorageInterface.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Interface used within QueryOutput, to handle diffrent ways we may store records + ** (e.g., in a list (that holds them all), or a pipe, that streams them to a consumer thread)) + *******************************************************************************/ +interface QueryOutputStorageInterface +{ + + /******************************************************************************* + ** add a records to this output + *******************************************************************************/ + void addRecord(QRecord record); + + + /******************************************************************************* + ** add a list of records to this output + *******************************************************************************/ + void addRecords(List records); + + /******************************************************************************* + ** Get all stored records + *******************************************************************************/ + List getRecords(); +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index 0c1849e0..ff3761f7 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -27,8 +27,8 @@ import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.Month; -import java.util.ArrayList; -import java.util.List; +import java.util.Objects; +import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; @@ -53,21 +53,24 @@ public class MockQueryAction implements QueryInterface { QTableMetaData table = queryInput.getTable(); - QueryOutput rs = new QueryOutput(); - List records = new ArrayList<>(); - rs.setRecords(records); + QueryOutput queryOutput = new QueryOutput(queryInput); - QRecord record = new QRecord(); - records.add(record); - record.setTableName(table.getName()); - - for(String field : table.getFields().keySet()) + int rows = Objects.requireNonNullElse(queryInput.getLimit(), 1); + for(int i = 0; i < rows; i++) { - Serializable value = getValue(table, field); - record.setValue(field, value); + QRecord record = new QRecord(); + record.setTableName(table.getName()); + + for(String field : table.getFields().keySet()) + { + Serializable value = field.equals("id") ? (i + 1) : getValue(table, field); + record.setValue(field, value); + } + + queryOutput.addRecord(record); } - return rs; + return (queryOutput); } catch(Exception e) { @@ -87,7 +90,7 @@ public class MockQueryAction implements QueryInterface // @formatter:off // IJ can't do new-style switch correctly yet... return switch(table.getField(field).getType()) { - case STRING -> "Foo"; + case STRING -> UUID.randomUUID().toString(); case INTEGER -> 42; case DECIMAL -> new BigDecimal("3.14159"); case DATE -> LocalDate.of(1970, Month.JANUARY, 1); diff --git a/src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java b/src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java index d9435524..9da763fb 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java @@ -27,6 +27,11 @@ package com.kingsrook.qqq.backend.core.state; *******************************************************************************/ public abstract class AbstractStateKey { + /******************************************************************************* + ** Make the key give a unique string to identify itself. + * + *******************************************************************************/ + public abstract String getUniqueIdentifier(); /******************************************************************************* ** Require all state keys to implement the equals method diff --git a/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java b/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java index 9b0d4390..ed6df69d 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java @@ -76,7 +76,7 @@ public class TempFileStateProvider implements StateProviderInterface try { String json = JsonUtils.toJson(data); - FileUtils.writeStringToFile(new File("/tmp/" + key.toString()), json); + FileUtils.writeStringToFile(getFile(key), json); } catch(IOException e) { @@ -95,7 +95,7 @@ public class TempFileStateProvider implements StateProviderInterface { try { - String json = FileUtils.readFileToString(new File("/tmp/" + key.toString())); + String json = FileUtils.readFileToString(getFile(key)); return (Optional.of(JsonUtils.toObject(json, type))); } catch(FileNotFoundException fnfe) @@ -109,4 +109,14 @@ public class TempFileStateProvider implements StateProviderInterface } } + + + /******************************************************************************* + ** Get the file referenced by a key + *******************************************************************************/ + private File getFile(AbstractStateKey key) + { + return new File("/tmp/" + key.getUniqueIdentifier()); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java b/src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java index 2dcc2bd4..5e4ecf09 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java @@ -81,6 +81,18 @@ public class UUIDAndTypeStateKey extends AbstractStateKey + /******************************************************************************* + ** Make the key give a unique string to identify itself. + * + *******************************************************************************/ + @Override + public String getUniqueIdentifier() + { + return (uuid.toString()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qqq/backend/core/utils/SleepUtils.java b/src/main/java/com/kingsrook/qqq/backend/core/utils/SleepUtils.java new file mode 100644 index 00000000..fc89d1da --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/utils/SleepUtils.java @@ -0,0 +1,58 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils; + + +import java.util.concurrent.TimeUnit; + + +/******************************************************************************* + ** Utility methods to help with sleeping! + *******************************************************************************/ +public class SleepUtils +{ + + /******************************************************************************* + ** Sleep for as close as we can to the specified amount of time (ignoring + ** InterruptedException - continuing to sleep more). + *******************************************************************************/ + public static void sleep(long duration, TimeUnit timeUnit) + { + long millis = timeUnit.toMillis(duration); + long start = System.currentTimeMillis(); + long end = start + millis; + + while(System.currentTimeMillis() < end) + { + try + { + long millisToSleep = end - System.currentTimeMillis(); + Thread.sleep(millisToSleep); + } + catch(InterruptedException e) + { + // sleep more. + } + } + } + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java b/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java new file mode 100644 index 00000000..c356f7ec --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java @@ -0,0 +1,153 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for the ReportAction + *******************************************************************************/ +class ReportActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCSV() throws Exception + { + int recordCount = 1000; + String filename = "/tmp/ReportActionTest.csv"; + + runReport(recordCount, filename, ReportFormat.CSV, false); + + File file = new File(filename); + List fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name()); + assertEquals(recordCount + 1, fileLines.size()); + assertTrue(file.delete()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testExcel() throws Exception + { + int recordCount = 1000; + String filename = "/tmp/ReportActionTest.xlsx"; + + runReport(recordCount, filename, ReportFormat.XLSX, true); + + File file = new File(filename); + + assertTrue(file.delete()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runReport(int recordCount, String filename, ReportFormat reportFormat, boolean specifyFields) throws IOException, QException + { + try(FileOutputStream outputStream = new FileOutputStream(filename)) + { + ReportInput reportInput = new ReportInput(TestUtils.defineInstance(), TestUtils.getMockSession()); + reportInput.setTableName("person"); + QTableMetaData table = reportInput.getTable(); + + reportInput.setReportFormat(reportFormat); + reportInput.setReportOutputStream(outputStream); + reportInput.setQueryFilter(new QQueryFilter()); + reportInput.setLimit(recordCount); + + if(specifyFields) + { + reportInput.setFieldNames(table.getFields().values().stream().map(QFieldMetaData::getName).collect(Collectors.toList())); + } + ReportOutput reportOutput = new ReportAction().execute(reportInput); + assertNotNull(reportOutput); + assertEquals(recordCount, reportOutput.getRecordCount()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadFieldNames() + { + ReportInput reportInput = new ReportInput(TestUtils.defineInstance(), TestUtils.getMockSession()); + reportInput.setTableName("person"); + reportInput.setFieldNames(List.of("Foo", "Bar", "Baz")); + assertThrows(QUserFacingException.class, () -> + { + new ReportAction().execute(reportInput); + }); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreExecuteCount() throws QException + { + ReportInput reportInput = new ReportInput(TestUtils.defineInstance(), TestUtils.getMockSession()); + reportInput.setTableName("person"); + + //////////////////////////////////////////////////////////////// + // use xlsx, which has a max-rows limit, to verify that code. // + //////////////////////////////////////////////////////////////// + reportInput.setReportFormat(ReportFormat.XLSX); + + new ReportAction().preExecute(reportInput); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/core/utils/SleepUtilsTest.java b/src/test/java/com/kingsrook/qqq/backend/core/utils/SleepUtilsTest.java new file mode 100644 index 00000000..68653439 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/core/utils/SleepUtilsTest.java @@ -0,0 +1,50 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils; + + +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for SleepUtils + *******************************************************************************/ +class SleepUtilsTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSleep() + { + long start = System.currentTimeMillis(); + SleepUtils.sleep(10, TimeUnit.MILLISECONDS); + long end = System.currentTimeMillis(); + long sleptFor = end - start; + System.out.println("Slept for: " + sleptFor); + assertTrue(sleptFor >= 10); + } + +} \ No newline at end of file