QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-26 Initial buildout of qqq exporting capability

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-19 10:36:29 -05:00
parent f35744cd19
commit 84b0e12493
26 changed files with 2026 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.circleci/config.yml
View File

@ -24,6 +24,8 @@ commands:
name: Run Maven
command: |
mvn -s .circleci/mvn-settings.xml << parameters.maven_subcommand >>
- store_artifacts:
path: target/site/jacoco
- run:
name: Save test results
command: |

1
.gitignore vendored
View File

@ -28,3 +28,4 @@ target/
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
.DS_Store

7
pom.xml
View File

@ -79,11 +79,18 @@
<version>3.23.1</version>
<scope>test</scope>
</dependency>
<!-- generates Method objects from method references, used initially by QFieldMetaData to make fields from getter method refs -->
<dependency>
<groupId>com.github.hervian</groupId>
<artifactId>safety-mirror</artifactId>
<version>4.0.1</version>
</dependency>
<!-- excel library -->
<dependency>
<groupId>org.dhatim</groupId>
<artifactId>fastexcel</artifactId>
<version>0.12.15</version>
</dependency>
<!-- Common deps for all qqq modules -->
<dependency>

118
src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java
View File

@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -49,10 +50,21 @@ public class AsyncJobManager
/*******************************************************************************
** Run a job - if it finishes within the specified timeout, get its results,
** Start a job - if it finishes within the specified timeout, get its results,
** else, get back an exception with the job id.
*******************************************************************************/
public <T extends Serializable> T startJob(long timeout, TimeUnit timeUnit, AsyncJob<T> asyncJob) throws JobGoingAsyncException, QException
{
return (startJob("Anonymous", timeout, timeUnit, asyncJob));
}
/*******************************************************************************
** Start a job - if it finishes within the specified timeout, get its results,
** else, get back an exception with the job id.
*******************************************************************************/
public <T extends Serializable> T startJob(String jobName, long timeout, TimeUnit timeUnit, AsyncJob<T> asyncJob) throws JobGoingAsyncException, QException
{
UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS);
AsyncJobStatus asyncJobStatus = new AsyncJobStatus();
@ -63,22 +75,7 @@ public class AsyncJobManager
{
CompletableFuture<T> future = CompletableFuture.supplyAsync(() ->
{
try
{
LOG.info("Starting job " + uuidAndTypeStateKey.getUuid());
T result = asyncJob.run(new AsyncJobCallback(uuidAndTypeStateKey.getUuid(), asyncJobStatus));
asyncJobStatus.setState(AsyncJobState.COMPLETE);
getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
LOG.info("Completed job " + uuidAndTypeStateKey.getUuid());
return (result);
}
catch(Exception e)
{
asyncJobStatus.setState(AsyncJobState.ERROR);
asyncJobStatus.setCaughtException(e);
getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
throw (new CompletionException(e));
}
return (runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
});
T result = future.get(timeout, timeUnit);
@ -97,6 +94,66 @@ public class AsyncJobManager
/*******************************************************************************
** Start a job, and always, just get back the job UUID.
*******************************************************************************/
public <T extends Serializable> String startJob(AsyncJob<T> asyncJob)
{
return (startJob("Anonymous", asyncJob));
}
/*******************************************************************************
** Start a job, and always, just get back the job UUID.
*******************************************************************************/
public <T extends Serializable> String startJob(String jobName, AsyncJob<T> asyncJob)
{
UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS);
AsyncJobStatus asyncJobStatus = new AsyncJobStatus();
asyncJobStatus.setState(AsyncJobState.RUNNING);
getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
// todo - refactor, share
CompletableFuture.supplyAsync(() -> runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
return (uuidAndTypeStateKey.getUuid().toString());
}
/*******************************************************************************
** run the job.
*******************************************************************************/
private <T extends Serializable> T runAsyncJob(String jobName, AsyncJob<T> asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus)
{
String originalThreadName = Thread.currentThread().getName();
Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8));
try
{
LOG.info("Starting job " + uuidAndTypeStateKey.getUuid());
T result = asyncJob.run(new AsyncJobCallback(uuidAndTypeStateKey.getUuid(), asyncJobStatus));
asyncJobStatus.setState(AsyncJobState.COMPLETE);
getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
LOG.info("Completed job " + uuidAndTypeStateKey.getUuid());
return (result);
}
catch(Exception e)
{
asyncJobStatus.setState(AsyncJobState.ERROR);
asyncJobStatus.setCaughtException(e);
getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
LOG.warn("Job " + uuidAndTypeStateKey.getUuid() + " ended with an exception: ", e);
throw (new CompletionException(e));
}
finally
{
Thread.currentThread().setName(originalThreadName);
}
}
/*******************************************************************************
** Get the status of the job identified by the given UUID.
**
@ -122,4 +179,31 @@ public class AsyncJobManager
// return TempFileStateProvider.getInstance();
}
/*******************************************************************************
**
*******************************************************************************/
public AsyncJobStatus waitForJob(String jobUUID) throws QException
{
AsyncJobState asyncJobState = AsyncJobState.RUNNING;
AsyncJobStatus asyncJobStatus = null;
while(asyncJobState.equals(AsyncJobState.RUNNING))
{
LOG.info("Sleeping, waiting on job [" + jobUUID + "]");
SleepUtils.sleep(100, TimeUnit.MILLISECONDS);
Optional<AsyncJobStatus> optionalAsyncJobStatus = getJobStatus(jobUUID);
if(optionalAsyncJobStatus.isEmpty())
{
// todo - ... maybe some version of try-again?
throw (new QException("Could not get status of report query job [" + jobUUID + "]"));
}
asyncJobStatus = optionalAsyncJobStatus.get();
asyncJobState = asyncJobStatus.getState();
}
return (asyncJobStatus);
}
}

143
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java Normal file
View File

@ -0,0 +1,143 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.kingsrook.qqq.backend.core.adapters.QRecordToCsvAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** CSV report format implementation
*******************************************************************************/
public class CsvReportStreamer implements ReportStreamerInterface
{
private static final Logger LOG = LogManager.getLogger(CsvReportStreamer.class);
private final QRecordToCsvAdapter qRecordToCsvAdapter;
private ReportInput reportInput;
private QTableMetaData table;
private List<QFieldMetaData> fields;
private OutputStream outputStream;
/*******************************************************************************
**
*******************************************************************************/
public CsvReportStreamer()
{
qRecordToCsvAdapter = new QRecordToCsvAdapter();
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void start(ReportInput reportInput) throws QReportingException
{
this.reportInput = reportInput;
table = reportInput.getTable();
outputStream = this.reportInput.getReportOutputStream();
fields = setupFieldList(table, reportInput);
writeReportHeaderRow();
}
/*******************************************************************************
**
*******************************************************************************/
private void writeReportHeaderRow() throws QReportingException
{
try
{
int col = 0;
for(QFieldMetaData column : fields)
{
if(col++ > 0)
{
outputStream.write(',');
}
outputStream.write(('"' + column.getLabel() + '"').getBytes(StandardCharsets.UTF_8));
}
outputStream.write('\n');
}
catch(Exception e)
{
throw (new QReportingException("Error starting CSV report"));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public int takeRecordsFromPipe(RecordPipe recordPipe) throws QReportingException
{
List<QRecord> qRecords = recordPipe.consumeAvailableRecords();
LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
try
{
for(QRecord qRecord : qRecords)
{
String csv = qRecordToCsvAdapter.recordToCsv(table, qRecord, fields);
outputStream.write(csv.getBytes(StandardCharsets.UTF_8));
outputStream.flush(); // todo - less often?
}
return (qRecords.size());
}
catch(Exception e)
{
throw (new QReportingException("Error writing CSV report", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void finish()
{
}
}

213
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java Normal file
View File

@ -0,0 +1,213 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.io.OutputStream;
import java.io.Serializable;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.util.Date;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dhatim.fastexcel.Workbook;
import org.dhatim.fastexcel.Worksheet;
/*******************************************************************************
** Excel report format implementation
*******************************************************************************/
public class ExcelReportStreamer implements ReportStreamerInterface
{
private static final Logger LOG = LogManager.getLogger(ExcelReportStreamer.class);
private ReportInput reportInput;
private QTableMetaData table;
private List<QFieldMetaData> fields;
private OutputStream outputStream;
private Workbook workbook;
private Worksheet worksheet;
private int row = 1;
/*******************************************************************************
**
*******************************************************************************/
public ExcelReportStreamer()
{
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void start(ReportInput reportInput) throws QReportingException
{
this.reportInput = reportInput;
table = reportInput.getTable();
outputStream = this.reportInput.getReportOutputStream();
workbook = new Workbook(outputStream, "QQQ", null);
worksheet = workbook.newWorksheet("Sheet 1");
fields = setupFieldList(table, reportInput);
writeReportHeaderRow();
}
/*******************************************************************************
**
*******************************************************************************/
private void writeReportHeaderRow() throws QReportingException
{
try
{
int col = 0;
for(QFieldMetaData column : fields)
{
worksheet.value(0, col, column.getLabel());
col++;
}
}
catch(Exception e)
{
throw (new QReportingException("Error starting Excel report"));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public int takeRecordsFromPipe(RecordPipe recordPipe) throws QReportingException
{
List<QRecord> qRecords = recordPipe.consumeAvailableRecords();
LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
try
{
for(QRecord qRecord : qRecords)
{
int col = 0;
for(QFieldMetaData column : fields)
{
Serializable value = qRecord.getValue(column.getName());
if(value != null)
{
if(value instanceof String s)
{
worksheet.value(row, col, s);
}
else if(value instanceof Number n)
{
worksheet.value(row, col, n);
}
else if(value instanceof Boolean b)
{
worksheet.value(row, col, b);
}
else if(value instanceof Date d)
{
worksheet.value(row, col, d);
worksheet.style(row, col).format("yyyy-MM-dd").set();
}
else if(value instanceof LocalDate d)
{
worksheet.value(row, col, d);
worksheet.style(row, col).format("yyyy-MM-dd").set();
}
else if(value instanceof LocalDateTime d)
{
worksheet.value(row, col, d);
worksheet.style(row, col).format("yyyy-MM-dd H:mm:ss").set();
}
else if(value instanceof ZonedDateTime d)
{
worksheet.value(row, col, d);
worksheet.style(row, col).format("yyyy-MM-dd H:mm:ss").set();
}
else
{
worksheet.value(row, col, ValueUtils.getValueAsString(value));
}
}
col++;
}
row++;
worksheet.flush(); // todo? not at all? or just sometimes?
}
}
catch(Exception e)
{
try
{
workbook.finish();
outputStream.close();
}
finally
{
throw (new QReportingException("Error generating Excel report", e));
}
}
return (qRecords.size());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void finish() throws QReportingException
{
try
{
if(workbook != null)
{
workbook.finish();
}
}
catch(Exception e)
{
throw (new QReportingException("Error finishing Excel report", e));
}
}
}

92
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java Normal file
View File

@ -0,0 +1,92 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Object to connect a producer of records with a consumer.
** Best for those to be on different threads, to avoid deadlock.
*******************************************************************************/
public class RecordPipe
{
private Queue<QRecord> queue = new ArrayDeque<>();
/*******************************************************************************
** Add a record to the pipe
*******************************************************************************/
public void addRecord(QRecord record)
{
queue.add(record);
}
/*******************************************************************************
** Add a list of records to the pipe
*******************************************************************************/
public void addRecords(List<QRecord> records)
{
queue.addAll(records);
}
/*******************************************************************************
**
*******************************************************************************/
public List<QRecord> consumeAvailableRecords()
{
List<QRecord> rs = new ArrayList<>();
while(true)
{
QRecord record = queue.poll();
if(record == null)
{
break;
}
rs.add(record);
}
return (rs);
}
/*******************************************************************************
**
*******************************************************************************/
public int countAvailableRecords()
{
return (queue.size());
}
}

265
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java Normal file
View File

@ -0,0 +1,265 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Action to generate a report.
**
** At this time (future may change?), this action starts a new thread to run
** the query in the backend module. As records are produced by the query,
** they are put into a RecordPipe, which the ReportStreamer pulls from, to write
** to the report output stream. This action will block until the query job
** is complete, and the final records have been consumed from the pipe, at which
** time the report outputStream can be closed.
**
*******************************************************************************/
public class ReportAction
{
private static final Logger LOG = LogManager.getLogger(ReportAction.class);
private boolean preExecuteRan = false;
/*******************************************************************************
** Validation logic, that will run before the action is executed -- ideally, if
** a caller is going to run the execution in a thread, they'd call this method
** first, in their thread, to catch any validation errors before they start
** the thread (which they may abandon).
*******************************************************************************/
public void preExecute(ReportInput reportInput) throws QException
{
ActionHelper.validateSession(reportInput);
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface backendModule = qBackendModuleDispatcher.getQBackendModule(reportInput.getBackend());
///////////////////////////////////
// verify field names (if given) //
///////////////////////////////////
if(CollectionUtils.nullSafeHasContents(reportInput.getFieldNames()))
{
QTableMetaData table = reportInput.getTable();
List<String> badFieldNames = new ArrayList<>();
for(String fieldName : reportInput.getFieldNames())
{
try
{
table.getField(fieldName);
}
catch(IllegalArgumentException iae)
{
badFieldNames.add(fieldName);
}
}
if(!badFieldNames.isEmpty())
{
throw (new QUserFacingException(badFieldNames.size() == 1
? ("Field name " + badFieldNames.get(0) + " was not found on the " + table.getLabel() + " table.")
: ("Fields names " + StringUtils.joinWithCommasAndAnd(badFieldNames) + " were not found on the " + table.getLabel() + " table.")));
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// check if this report format has a max-rows limit -- if so, do a count to verify we're under the limit //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
ReportFormat reportFormat = reportInput.getReportFormat();
verifyCountUnderMax(reportInput, backendModule, reportFormat);
preExecuteRan = true;
}
/*******************************************************************************
** Run the report.
*******************************************************************************/
public ReportOutput execute(ReportInput reportInput) throws QException
{
if(!preExecuteRan)
{
/////////////////////////////////////
// ensure that pre-execute has ran //
/////////////////////////////////////
preExecute(reportInput);
}
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface backendModule = qBackendModuleDispatcher.getQBackendModule(reportInput.getBackend());
QTableMetaData table = reportInput.getTable();
//////////////////////////
// set up a query input //
//////////////////////////
QueryInterface queryInterface = backendModule.getQueryInterface();
QueryInput queryInput = new QueryInput(reportInput.getInstance());
queryInput.setSession(reportInput.getSession());
queryInput.setTableName(reportInput.getTableName());
queryInput.setFilter(reportInput.getQueryFilter());
queryInput.setLimit(reportInput.getLimit());
/////////////////////////////////////////////////////////////////
// tell this query that it needs to put its output into a pipe //
/////////////////////////////////////////////////////////////////
RecordPipe recordPipe = new RecordPipe();
queryInput.setRecordPipe(recordPipe);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set up a report streamer, which will read rows from the pipe, and write formatted report rows to the output stream //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ReportFormat reportFormat = reportInput.getReportFormat();
ReportStreamerInterface reportStreamer = reportFormat.newReportStreamer();
reportStreamer.start(reportInput);
//////////////////////////////////////////
// run the query action as an async job //
//////////////////////////////////////////
AsyncJobManager asyncJobManager = new AsyncJobManager();
String jobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput)));
LOG.info("Started query job [" + jobUUID + "] for report");
AsyncJobState asyncJobState = AsyncJobState.RUNNING;
AsyncJobStatus asyncJobStatus = null;
int nextSleepMillis = 10;
long recordCount = 0;
while(asyncJobState.equals(AsyncJobState.RUNNING))
{
int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
recordCount += recordsConsumed;
if(recordsConsumed == 0)
{
//////////////////////////////////////////////////////////////////////////////////
// do we need to sleep to let the producer work? //
// todo - smarter sleep? like an exponential back-off? and eventually a fail? //
//////////////////////////////////////////////////////////////////////////////////
LOG.info("Read 0 records from pipe, sleeping to give producer a chance to work");
SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
while(recordPipe.countAvailableRecords() == 0)
{
nextSleepMillis *= 2;
LOG.info("Still no records in the pipe, so sleeping more [" + nextSleepMillis + "]ms");
SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
}
}
nextSleepMillis = 10;
Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(jobUUID);
if(optionalAsyncJobStatus.isEmpty())
{
/////////////////////////////////////////////////
// todo - ... maybe some version of try-again? //
/////////////////////////////////////////////////
throw (new QException("Could not get status of report query job [" + jobUUID + "]"));
}
asyncJobStatus = optionalAsyncJobStatus.get();
asyncJobState = asyncJobStatus.getState();
}
LOG.info("Query job [" + jobUUID + "] for report completed with status: " + asyncJobStatus);
///////////////////////////////////////////////////
// send the final records to the report streamer //
///////////////////////////////////////////////////
int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
recordCount += recordsConsumed;
//////////////////////////////////////////////////////////////////
// Critical: we must close the stream here as our final action //
//////////////////////////////////////////////////////////////////
reportStreamer.finish();
try
{
reportInput.getReportOutputStream().close();
}
catch(Exception e)
{
throw (new QReportingException("Error completing report", e));
}
ReportOutput reportOutput = new ReportOutput();
reportOutput.setRecordCount(recordCount);
return (reportOutput);
}
/*******************************************************************************
**
*******************************************************************************/
private void verifyCountUnderMax(ReportInput reportInput, QBackendModuleInterface backendModule, ReportFormat reportFormat) throws QException
{
if(reportFormat.getMaxRows() != null)
{
if(reportInput.getLimit() == null || reportInput.getLimit() > reportFormat.getMaxRows())
{
CountInterface countInterface = backendModule.getCountInterface();
CountInput countInput = new CountInput(reportInput.getInstance());
countInput.setSession(reportInput.getSession());
countInput.setTableName(reportInput.getTableName());
countInput.setFilter(reportInput.getQueryFilter());
CountOutput countOutput = countInterface.execute(countInput);
Integer count = countOutput.getCount();
if(count > reportFormat.getMaxRows())
{
throw (new QUserFacingException("The requested report would include more rows ("
+ String.format("%,d", count) + ") than the maximum allowed ("
+ String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
}
}
}
}
}

69
src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java Normal file
View File

@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Interface for various report formats to implement.
*******************************************************************************/
public interface ReportStreamerInterface
{
/*******************************************************************************
** Called once, before any rows are available. Meant to write a header, for example.
*******************************************************************************/
void start(ReportInput reportInput) throws QReportingException;
/*******************************************************************************
** Called as records flow into the pipe.
******************************************************************************/
int takeRecordsFromPipe(RecordPipe recordPipe) throws QReportingException;
/*******************************************************************************
** Called once, after all rows are available. Meant to write a footer, or close resources, for example.
*******************************************************************************/
void finish() throws QReportingException;
/*******************************************************************************
** (Ideally, protected) method used within report streamer implementations, to
** map field names from reportInput into list of fieldMetaData.
*******************************************************************************/
default List<QFieldMetaData> setupFieldList(QTableMetaData table, ReportInput reportInput)
{
if(reportInput.getFieldNames() != null)
{
return (reportInput.getFieldNames().stream().map(table::getField).toList());
}
else
{
return (new ArrayList<>(table.getFields().values()));
}
}
}

87
src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java Normal file
View File

@ -0,0 +1,87 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.adapters;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** Class to convert QRecords to CSV Strings.
*******************************************************************************/
public class QRecordToCsvAdapter
{
/*******************************************************************************
**
*******************************************************************************/
public String recordToCsv(QTableMetaData table, QRecord record)
{
return (recordToCsv(table, record, new ArrayList<>(table.getFields().values())));
}
/*******************************************************************************
**
*******************************************************************************/
public String recordToCsv(QTableMetaData table, QRecord record, List<QFieldMetaData> fields)
{
StringBuilder rs = new StringBuilder();
int fieldNo = 0;
for(QFieldMetaData field : fields)
{
if(fieldNo++ > 0)
{
rs.append(',');
}
rs.append('"');
Serializable value = record.getValue(field.getName());
String valueAsString = ValueUtils.getValueAsString(value);
if(StringUtils.hasContent(valueAsString))
{
rs.append(sanitize(valueAsString));
}
rs.append('"');
}
rs.append('\n');
return (rs.toString());
}
/*******************************************************************************
** todo - kinda weak... can we find this in a CSV lib??
*******************************************************************************/
private String sanitize(String value)
{
return (value.replaceAll("\"", "\"\"").replaceAll("\n", " "));
}
}

51
src/main/java/com/kingsrook/qqq/backend/core/exceptions/QReportingException.java Normal file
View File

@ -0,0 +1,51 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.exceptions;
/*******************************************************************************
* Exception thrown while generating reports
*
*******************************************************************************/
public class QReportingException extends QException
{
/*******************************************************************************
** Constructor of message
**
*******************************************************************************/
public QReportingException(String message)
{
super(message);
}
/*******************************************************************************
** Constructor of message & cause
**
*******************************************************************************/
public QReportingException(String message, Throwable cause)
{
super(message, cause);
}
}

114
src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java Normal file
View File

@ -0,0 +1,114 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.reporting;
import java.util.Locale;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.actions.reporting.CsvReportStreamer;
import com.kingsrook.qqq.backend.core.actions.reporting.ExcelReportStreamer;
import com.kingsrook.qqq.backend.core.actions.reporting.ReportStreamerInterface;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.dhatim.fastexcel.Worksheet;
/*******************************************************************************
** QQQ Report/export file formats
*******************************************************************************/
public enum ReportFormat
{
XLSX(Worksheet.MAX_ROWS, ExcelReportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
CSV(null, CsvReportStreamer::new, "text/csv");
private final Integer maxRows;
private final String mimeType;
private final Supplier<? extends ReportStreamerInterface> streamerConstructor;
/*******************************************************************************
**
*******************************************************************************/
ReportFormat(Integer maxRows, Supplier<? extends ReportStreamerInterface> streamerConstructor, String mimeType)
{
this.maxRows = maxRows;
this.mimeType = mimeType;
this.streamerConstructor = streamerConstructor;
}
/*******************************************************************************
**
*******************************************************************************/
public static ReportFormat fromString(String format) throws QUserFacingException
{
if(!StringUtils.hasContent(format))
{
throw (new QUserFacingException("Report format was not specified."));
}
try
{
return (ReportFormat.valueOf(format.toUpperCase(Locale.ROOT)));
}
catch(IllegalArgumentException iae)
{
throw (new QUserFacingException("Unsupported report format: " + format + "."));
}
}
/*******************************************************************************
** Getter for maxRows
**
*******************************************************************************/
public Integer getMaxRows()
{
return maxRows;
}
/*******************************************************************************
** Getter for mimeType
**
*******************************************************************************/
public String getMimeType()
{
return mimeType;
}
/*******************************************************************************
**
*******************************************************************************/
public ReportStreamerInterface newReportStreamer()
{
return (streamerConstructor.get());
}
}

207
src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java Normal file
View File

@ -0,0 +1,207 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.reporting;
import java.io.OutputStream;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
/*******************************************************************************
** Input for a Report action
*******************************************************************************/
public class ReportInput extends AbstractTableActionInput
{
private QQueryFilter queryFilter;
private Integer limit;
private List<String> fieldNames;
private String filename;
private ReportFormat reportFormat;
private OutputStream reportOutputStream;
/*******************************************************************************
**
*******************************************************************************/
public ReportInput()
{
}
/*******************************************************************************
**
*******************************************************************************/
public ReportInput(QInstance instance)
{
super(instance);
}
/*******************************************************************************
**
*******************************************************************************/
public ReportInput(QInstance instance, QSession session)
{
super(instance);
setSession(session);
}
/*******************************************************************************
** Getter for queryFilter
**
*******************************************************************************/
public QQueryFilter getQueryFilter()
{
return queryFilter;
}
/*******************************************************************************
** Setter for queryFilter
**
*******************************************************************************/
public void setQueryFilter(QQueryFilter queryFilter)
{
this.queryFilter = queryFilter;
}
/*******************************************************************************
** Getter for limit
**
*******************************************************************************/
public Integer getLimit()
{
return limit;
}
/*******************************************************************************
** Setter for limit
**
*******************************************************************************/
public void setLimit(Integer limit)
{
this.limit = limit;
}
/*******************************************************************************
** Getter for fieldNames
**
*******************************************************************************/
public List<String> getFieldNames()
{
return fieldNames;
}
/*******************************************************************************
** Setter for fieldNames
**
*******************************************************************************/
public void setFieldNames(List<String> fieldNames)
{
this.fieldNames = fieldNames;
}
/*******************************************************************************
** Getter for filename
**
*******************************************************************************/
public String getFilename()
{
return filename;
}
/*******************************************************************************
** Setter for filename
**
*******************************************************************************/
public void setFilename(String filename)
{
this.filename = filename;
}
/*******************************************************************************
** Getter for reportFormat
**
*******************************************************************************/
public ReportFormat getReportFormat()
{
return reportFormat;
}
/*******************************************************************************
** Setter for reportFormat
**
*******************************************************************************/
public void setReportFormat(ReportFormat reportFormat)
{
this.reportFormat = reportFormat;
}
/*******************************************************************************
** Getter for reportOutputStream
**
*******************************************************************************/
public OutputStream getReportOutputStream()
{
return reportOutputStream;
}
/*******************************************************************************
** Setter for reportOutputStream
**
*******************************************************************************/
public void setReportOutputStream(OutputStream reportOutputStream)
{
this.reportOutputStream = reportOutputStream;
}
}

56
src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java Normal file
View File

@ -0,0 +1,56 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.reporting;
import java.io.Serializable;
/*******************************************************************************
** Output for a Report action
*******************************************************************************/
public class ReportOutput implements Serializable
{
public long recordCount;
/*******************************************************************************
** Getter for recordCount
**
*******************************************************************************/
public long getRecordCount()
{
return recordCount;
}
/*******************************************************************************
** Setter for recordCount
**
*******************************************************************************/
public void setRecordCount(long recordCount)
{
this.recordCount = recordCount;
}
}

26
src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -36,6 +37,8 @@ public class QueryInput extends AbstractTableActionInput
private Integer skip;
private Integer limit;
private RecordPipe recordPipe;
/*******************************************************************************
@ -120,4 +123,27 @@ public class QueryInput extends AbstractTableActionInput
{
this.limit = limit;
}
/*******************************************************************************
** Getter for recordPipe
**
*******************************************************************************/
public RecordPipe getRecordPipe()
{
return recordPipe;
}
/*******************************************************************************
** Setter for recordPipe
**
*******************************************************************************/
public void setRecordPipe(RecordPipe recordPipe)
{
this.recordPipe = recordPipe;
}
}

59
src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -31,9 +32,52 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
** Output for a query action
**
*******************************************************************************/
public class QueryOutput extends AbstractActionOutput
public class QueryOutput extends AbstractActionOutput implements Serializable
{
private List<QRecord> records;
private QueryOutputStorageInterface storage;
/*******************************************************************************
** Construct a new query output, based on a query input (which will drive some
** of how our output is structured... e.g., if we pipe the output)
*******************************************************************************/
public QueryOutput(QueryInput queryInput)
{
if(queryInput.getRecordPipe() != null)
{
storage = new QueryOutputRecordPipe(queryInput.getRecordPipe());
}
else
{
storage = new QueryOutputList();
}
}
/*******************************************************************************
** Add a record to this output. Note - we often don't care, in such a method,
** whether the record is "completed" or not (e.g., all of its values have been
** populated) - but - note in here - that this records MAY be going into a pipe
** that could be read asynchronously, at any time, by another thread - SO - only
** completely populated records should be passed into this method.
*******************************************************************************/
public void addRecord(QRecord record)
{
storage.addRecord(record);
}
/*******************************************************************************
** add a list of records to this output
*******************************************************************************/
public void addRecords(List<QRecord> records)
{
storage.addRecords(records);
}
/*******************************************************************************
@ -41,16 +85,7 @@ public class QueryOutput extends AbstractActionOutput
*******************************************************************************/
public List<QRecord> getRecords()
{
return records;
return storage.getRecords();
}
/*******************************************************************************
**
*******************************************************************************/
public void setRecords(List<QRecord> records)
{
this.records = records;
}
}

80
src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputList.java Normal file
View File

@ -0,0 +1,80 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Kinda the standard way that a QueryOutput would store its records - in a
** simple list.
*******************************************************************************/
class QueryOutputList implements QueryOutputStorageInterface
{
private List<QRecord> records = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
public QueryOutputList()
{
}
/*******************************************************************************
** add a record to this output
*******************************************************************************/
@Override
public void addRecord(QRecord record)
{
records.add(record);
}
/*******************************************************************************
** add a list of records to this output
*******************************************************************************/
@Override
public void addRecords(List<QRecord> records)
{
this.records.addAll(records);
}
/*******************************************************************************
** Get all stored records
*******************************************************************************/
@Override
public List<QRecord> getRecords()
{
return (records);
}
}

107
src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java Normal file
View File

@ -0,0 +1,107 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Query output that uses a RecordPipe
*******************************************************************************/
class QueryOutputRecordPipe implements QueryOutputStorageInterface
{
private static final Logger LOG = LogManager.getLogger(QueryOutputRecordPipe.class);
private RecordPipe recordPipe;
/*******************************************************************************
**
*******************************************************************************/
public QueryOutputRecordPipe(RecordPipe recordPipe)
{
this.recordPipe = recordPipe;
}
/*******************************************************************************
** add a record to this output
*******************************************************************************/
@Override
public void addRecord(QRecord record)
{
recordPipe.addRecord(record);
blockIfPipeIsTooFull();
}
/*******************************************************************************
**
*******************************************************************************/
private void blockIfPipeIsTooFull()
{
if(recordPipe.countAvailableRecords() >= 100_000)
{
LOG.info("Record pipe is kinda full. Blocking for a bit");
do
{
SleepUtils.sleep(10, TimeUnit.MILLISECONDS);
}
while(recordPipe.countAvailableRecords() >= 10_000);
LOG.info("Done blocking.");
}
}
/*******************************************************************************
** add a list of records to this output
*******************************************************************************/
@Override
public void addRecords(List<QRecord> records)
{
recordPipe.addRecords(records);
blockIfPipeIsTooFull();
}
/*******************************************************************************
** Get all stored records
*******************************************************************************/
@Override
public List<QRecord> getRecords()
{
throw (new IllegalStateException("getRecords may not be called on a piped query output"));
}
}

51
src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputStorageInterface.java Normal file
View File

@ -0,0 +1,51 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Interface used within QueryOutput, to handle diffrent ways we may store records
** (e.g., in a list (that holds them all), or a pipe, that streams them to a consumer thread))
*******************************************************************************/
interface QueryOutputStorageInterface
{
/*******************************************************************************
** add a records to this output
*******************************************************************************/
void addRecord(QRecord record);
/*******************************************************************************
** add a list of records to this output
*******************************************************************************/
void addRecords(List<QRecord> records);
/*******************************************************************************
** Get all stored records
*******************************************************************************/
List<QRecord> getRecords();
}

31
src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java
View File

@ -27,8 +27,8 @@ import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.Month;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
@ -53,21 +53,24 @@ public class MockQueryAction implements QueryInterface
{
QTableMetaData table = queryInput.getTable();
QueryOutput rs = new QueryOutput();
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
QueryOutput queryOutput = new QueryOutput(queryInput);
QRecord record = new QRecord();
records.add(record);
record.setTableName(table.getName());
for(String field : table.getFields().keySet())
int rows = Objects.requireNonNullElse(queryInput.getLimit(), 1);
for(int i = 0; i < rows; i++)
{
Serializable value = getValue(table, field);
record.setValue(field, value);
QRecord record = new QRecord();
record.setTableName(table.getName());
for(String field : table.getFields().keySet())
{
Serializable value = field.equals("id") ? (i + 1) : getValue(table, field);
record.setValue(field, value);
}
queryOutput.addRecord(record);
}
return rs;
return (queryOutput);
}
catch(Exception e)
{
@ -87,7 +90,7 @@ public class MockQueryAction implements QueryInterface
// @formatter:off // IJ can't do new-style switch correctly yet...
return switch(table.getField(field).getType())
{
case STRING -> "Foo";
case STRING -> UUID.randomUUID().toString();
case INTEGER -> 42;
case DECIMAL -> new BigDecimal("3.14159");
case DATE -> LocalDate.of(1970, Month.JANUARY, 1);

5
src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java
View File

@ -27,6 +27,11 @@ package com.kingsrook.qqq.backend.core.state;
*******************************************************************************/
public abstract class AbstractStateKey
{
/*******************************************************************************
** Make the key give a unique string to identify itself.
*
*******************************************************************************/
public abstract String getUniqueIdentifier();
/*******************************************************************************
** Require all state keys to implement the equals method

14
src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java
View File

@ -76,7 +76,7 @@ public class TempFileStateProvider implements StateProviderInterface
try
{
String json = JsonUtils.toJson(data);
FileUtils.writeStringToFile(new File("/tmp/" + key.toString()), json);
FileUtils.writeStringToFile(getFile(key), json);
}
catch(IOException e)
{
@ -95,7 +95,7 @@ public class TempFileStateProvider implements StateProviderInterface
{
try
{
String json = FileUtils.readFileToString(new File("/tmp/" + key.toString()));
String json = FileUtils.readFileToString(getFile(key));
return (Optional.of(JsonUtils.toObject(json, type)));
}
catch(FileNotFoundException fnfe)
@ -109,4 +109,14 @@ public class TempFileStateProvider implements StateProviderInterface
}
}
/*******************************************************************************
** Get the file referenced by a key
*******************************************************************************/
private File getFile(AbstractStateKey key)
{
return new File("/tmp/" + key.getUniqueIdentifier());
}
}

12
src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java
View File

@ -81,6 +81,18 @@ public class UUIDAndTypeStateKey extends AbstractStateKey
/*******************************************************************************
** Make the key give a unique string to identify itself.
*
*******************************************************************************/
@Override
public String getUniqueIdentifier()
{
return (uuid.toString());
}
/*******************************************************************************
**
*******************************************************************************/

58
src/main/java/com/kingsrook/qqq/backend/core/utils/SleepUtils.java Normal file
View File

@ -0,0 +1,58 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.utils;
import java.util.concurrent.TimeUnit;
/*******************************************************************************
** Utility methods to help with sleeping!
*******************************************************************************/
public class SleepUtils
{
/*******************************************************************************
** Sleep for as close as we can to the specified amount of time (ignoring
** InterruptedException - continuing to sleep more).
*******************************************************************************/
public static void sleep(long duration, TimeUnit timeUnit)
{
long millis = timeUnit.toMillis(duration);
long start = System.currentTimeMillis();
long end = start + millis;
while(System.currentTimeMillis() < end)
{
try
{
long millisToSleep = end - System.currentTimeMillis();
Thread.sleep(millisToSleep);
}
catch(InterruptedException e)
{
// sleep more.
}
}
}
}

153
src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java Normal file
View File

@ -0,0 +1,153 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for the ReportAction
*******************************************************************************/
class ReportActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCSV() throws Exception
{
int recordCount = 1000;
String filename = "/tmp/ReportActionTest.csv";
runReport(recordCount, filename, ReportFormat.CSV, false);
File file = new File(filename);
List<String> fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name());
assertEquals(recordCount + 1, fileLines.size());
assertTrue(file.delete());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testExcel() throws Exception
{
int recordCount = 1000;
String filename = "/tmp/ReportActionTest.xlsx";
runReport(recordCount, filename, ReportFormat.XLSX, true);
File file = new File(filename);
assertTrue(file.delete());
}
/*******************************************************************************
**
*******************************************************************************/
private void runReport(int recordCount, String filename, ReportFormat reportFormat, boolean specifyFields) throws IOException, QException
{
try(FileOutputStream outputStream = new FileOutputStream(filename))
{
ReportInput reportInput = new ReportInput(TestUtils.defineInstance(), TestUtils.getMockSession());
reportInput.setTableName("person");
QTableMetaData table = reportInput.getTable();
reportInput.setReportFormat(reportFormat);
reportInput.setReportOutputStream(outputStream);
reportInput.setQueryFilter(new QQueryFilter());
reportInput.setLimit(recordCount);
if(specifyFields)
{
reportInput.setFieldNames(table.getFields().values().stream().map(QFieldMetaData::getName).collect(Collectors.toList()));
}
ReportOutput reportOutput = new ReportAction().execute(reportInput);
assertNotNull(reportOutput);
assertEquals(recordCount, reportOutput.getRecordCount());
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBadFieldNames()
{
ReportInput reportInput = new ReportInput(TestUtils.defineInstance(), TestUtils.getMockSession());
reportInput.setTableName("person");
reportInput.setFieldNames(List.of("Foo", "Bar", "Baz"));
assertThrows(QUserFacingException.class, () ->
{
new ReportAction().execute(reportInput);
});
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testPreExecuteCount() throws QException
{
ReportInput reportInput = new ReportInput(TestUtils.defineInstance(), TestUtils.getMockSession());
reportInput.setTableName("person");
////////////////////////////////////////////////////////////////
// use xlsx, which has a max-rows limit, to verify that code. //
////////////////////////////////////////////////////////////////
reportInput.setReportFormat(ReportFormat.XLSX);
new ReportAction().preExecute(reportInput);
}
}

50
src/test/java/com/kingsrook/qqq/backend/core/utils/SleepUtilsTest.java Normal file
View File

@ -0,0 +1,50 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.utils;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for SleepUtils
*******************************************************************************/
class SleepUtilsTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSleep()
{
long start = System.currentTimeMillis();
SleepUtils.sleep(10, TimeUnit.MILLISECONDS);
long end = System.currentTimeMillis();
long sleptFor = end - start;
System.out.println("Slept for: " + sleptFor);
assertTrue(sleptFor >= 10);
}
}