diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 34e33336..f93cbf50 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -80,6 +80,7 @@ public class RunProcessAction // indicator that the timestamp field should be updated - e.g., the execute step is finished. // //////////////////////////////////////////////////////////////////////////////////////////////// public static final String BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD = "basepullReadyToUpdateTimestamp"; + public static final String BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD = "basepullDidQueryUsingTimestamp"; @@ -190,11 +191,14 @@ public class RunProcessAction } } - //////////////////////////////////////////////////////////////////////////////////// - // if 'basepull' style process, update the stored basepull timestamp // - // but only when we've been signaled to do so - i.e., after an Execute step runs. // - //////////////////////////////////////////////////////////////////////////////////// - if(basepullConfiguration != null && BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD)))) + /////////////////////////////////////////////////////////////////////////// + // if 'basepull' style process, update the stored basepull timestamp // + // but only when we've been signaled to do so - i.e., only if we did our // + // query using the timestamp field, and only after an Execute step runs. // + /////////////////////////////////////////////////////////////////////////// + if(basepullConfiguration != null + && BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD))) + && BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD)))) { storeLastRunTime(runProcessInput, process, basepullConfiguration); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index 668217ce..8f951431 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -179,6 +179,17 @@ public class GenerateReportAction } outputSummaries(reportInput); + + reportStreamer.finish(); + + try + { + reportInput.getReportOutputStream().close(); + } + catch(Exception e) + { + throw (new QReportingException("Error completing report", e)); + } } @@ -527,8 +538,6 @@ public class GenerateReportAction reportStreamer.addTotalsRow(summaryOutput.totalRow); } } - - reportStreamer.finish(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java new file mode 100644 index 00000000..c58f2930 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java @@ -0,0 +1,155 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** JSON export format implementation + *******************************************************************************/ +public class JsonExportStreamer implements ExportStreamerInterface +{ + private static final Logger LOG = LogManager.getLogger(JsonExportStreamer.class); + + private ExportInput exportInput; + private QTableMetaData table; + private List fields; + private OutputStream outputStream; + + private boolean needComma = false; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public JsonExportStreamer() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void start(ExportInput exportInput, List fields, String label) throws QReportingException + { + this.exportInput = exportInput; + this.fields = fields; + table = exportInput.getTable(); + outputStream = this.exportInput.getReportOutputStream(); + + try + { + outputStream.write("[".getBytes(StandardCharsets.UTF_8)); + } + catch(IOException e) + { + throw (new QReportingException("Error starting report output", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addRecords(List qRecords) throws QReportingException + { + LOG.info("Consuming [" + qRecords.size() + "] records from the pipe"); + + for(QRecord qRecord : qRecords) + { + writeRecord(qRecord); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeRecord(QRecord qRecord) throws QReportingException + { + try + { + if(needComma) + { + outputStream.write(",".getBytes(StandardCharsets.UTF_8)); + } + + String json = JsonUtils.toJson(qRecord); + outputStream.write(json.getBytes(StandardCharsets.UTF_8)); + outputStream.flush(); // todo - less often? + needComma = true; + } + catch(Exception e) + { + throw (new QReportingException("Error writing JSON report", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addTotalsRow(QRecord record) throws QReportingException + { + writeRecord(record); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void finish() throws QReportingException + { + try + { + outputStream.write("]".getBytes(StandardCharsets.UTF_8)); + } + catch(IOException e) + { + throw (new QReportingException("Error ending report output", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java index db855917..3f6e6c95 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java @@ -38,7 +38,7 @@ import org.apache.logging.log4j.Logger; ** Base input class for all Q actions. ** *******************************************************************************/ -public abstract class AbstractActionInput +public class AbstractActionInput { private static final Logger LOG = LogManager.getLogger(AbstractActionInput.class); @@ -69,6 +69,17 @@ public abstract class AbstractActionInput + /******************************************************************************* + ** + *******************************************************************************/ + public AbstractActionInput(QInstance instance, QSession session) + { + this(instance); + this.session = session; + } + + + /******************************************************************************* ** performance instance validation (if not previously done). *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java index 9b41121d..c95b1be8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java @@ -27,6 +27,7 @@ import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.reporting.CsvExportStreamer; import com.kingsrook.qqq.backend.core.actions.reporting.ExcelExportStreamer; import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface; +import com.kingsrook.qqq.backend.core.actions.reporting.JsonExportStreamer; import com.kingsrook.qqq.backend.core.actions.reporting.ListOfMapsExportStreamer; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -39,6 +40,7 @@ import org.dhatim.fastexcel.Worksheet; public enum ReportFormat { XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelExportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), + JSON(null, null, JsonExportStreamer::new, "application/json"), CSV(null, null, CsvExportStreamer::new, "text/csv"), LIST_OF_MAPS(null, null, ListOfMapsExportStreamer::new, null); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java index d86bb5dd..60b7af34 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.reporting; import java.io.OutputStream; import java.io.Serializable; +import java.util.HashMap; import java.util.Map; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -118,6 +119,20 @@ public class ReportInput extends AbstractTableActionInput + /******************************************************************************* + ** + *******************************************************************************/ + public void addInputValue(String key, Serializable value) + { + if(this.inputValues == null) + { + this.inputValues = new HashMap<>(); + } + this.inputValues.put(key, value); + } + + + /******************************************************************************* ** Getter for filename ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java index e0df08c6..56402053 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -104,38 +104,7 @@ public class QFrontendTableMetaData Set enabledCapabilities = new HashSet<>(); for(Capability capability : Capability.values()) { - /////////////////////////////////////////////// - // by default, every table can do everything // - /////////////////////////////////////////////// - boolean hasCapability = true; - - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if the table's backend says the capability is disabled, then by default, then the capability is disabled... // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(backend.getDisabledCapabilities().contains(capability)) - { - hasCapability = false; - - ///////////////////////////////////////////////////////////////// - // unless the table overrides that and says that it IS enabled // - ///////////////////////////////////////////////////////////////// - if(table.getEnabledCapabilities().contains(capability)) - { - hasCapability = true; - } - } - else - { - ///////////////////////////////////////////////////////////////////////////////////////// - // if the backend doesn't specify the capability, then disable it if the table says so // - ///////////////////////////////////////////////////////////////////////////////////////// - if(table.getDisabledCapabilities().contains(capability)) - { - hasCapability = false; - } - } - - if(hasCapability) + if(table.isCapabilityEnabled(backend, capability)) { /////////////////////////////////////// // todo - check if user is allowed!! // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java index cc99b14d..dc3dbf3e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes; import java.io.Serializable; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; /******************************************************************************* @@ -65,4 +66,16 @@ public class AbstractProcessMetaDataBuilder .filter(f -> f.getName().equals(fieldName)).findFirst() .ifPresent(f -> f.setDefaultValue(value)); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public AbstractProcessMetaDataBuilder withBasepullConfiguration(BasepullConfiguration basepullConfiguration) + { + processMetaData.setBasepullConfiguration(basepullConfiguration); + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java index e8321397..cc0b8f56 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java @@ -34,8 +34,8 @@ public enum Capability TABLE_INSERT, TABLE_UPDATE, TABLE_DELETE - ////////////////////////////////////////////////////////////////////////// - // keep these values in sync with AdornmentType.ts in qqq-frontend-core // - ////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + // keep these values in sync with Capability.ts in qqq-frontend-core // + /////////////////////////////////////////////////////////////////////// } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java index caeff6e4..bccda36e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java @@ -41,6 +41,7 @@ public class QFieldSection private QIcon icon; private boolean isHidden = false; + private Integer gridColumns; @@ -328,4 +329,38 @@ public class QFieldSection return (this); } + + + /******************************************************************************* + ** Getter for gridColumns + ** + *******************************************************************************/ + public Integer getGridColumns() + { + return gridColumns; + } + + + + /******************************************************************************* + ** Setter for gridColumns + ** + *******************************************************************************/ + public void setGridColumns(Integer gridColumns) + { + this.gridColumns = gridColumns; + } + + + + /******************************************************************************* + ** Fluent setter for gridColumns + ** + *******************************************************************************/ + public QFieldSection withGridColumns(Integer gridColumns) + { + this.gridColumns = gridColumns; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 1530576c..d931f931 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.data.QRecordEntityField; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; @@ -1037,4 +1038,44 @@ public class QTableMetaData implements QAppChildMetaData, Serializable return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean isCapabilityEnabled(QBackendMetaData backend, Capability capability) + { + /////////////////////////////////////////////// + // by default, every table can do everything // + /////////////////////////////////////////////// + boolean hasCapability = true; + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the table's backend says the capability is disabled, then by default, then the capability is disabled... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(backend.getDisabledCapabilities().contains(capability)) + { + hasCapability = false; + + ///////////////////////////////////////////////////////////////// + // unless the table overrides that and says that it IS enabled // + ///////////////////////////////////////////////////////////////// + if(getEnabledCapabilities().contains(capability)) + { + hasCapability = true; + } + } + else + { + ///////////////////////////////////////////////////////////////////////////////////////// + // if the backend doesn't specify the capability, then disable it if the table says so // + ///////////////////////////////////////////////////////////////////////////////////////// + if(getDisabledCapabilities().contains(capability)) + { + hasCapability = false; + } + } + + return (hasCapability); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java index 806483af..722885ec 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java @@ -51,6 +51,7 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep try { queryFilter = super.getQueryFilter(runBackendStepInput); + return (queryFilter); } catch(QException qe) { @@ -77,6 +78,12 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep queryFilter.addOrderBy(new QFilterOrderBy(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD))); + ///////////////////////////////////////////////////////////////////////////////////// + // put a flag in the process's values, to note that we did use the timestamp field // + // this will later be checked to see if we should update the timestamp too. // + ///////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.addValue(RunProcessAction.BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD, true); + return (queryFilter); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java deleted file mode 100644 index 0219acf6..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; - - -import java.util.ArrayList; -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - - -/******************************************************************************* - ** Function body for performing the Load step of a basic ETL process using update. - *******************************************************************************/ -public class BasicETLLoadAsUpdateFunction implements BackendStep -{ - private static final Logger LOG = LogManager.getLogger(BasicETLLoadAsUpdateFunction.class); - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - ////////////////////////////////////////////////////// - // exit early with no-op if no records made it here // - ////////////////////////////////////////////////////// - List inputRecords = runBackendStepInput.getRecords(); - LOG.info("Received [" + inputRecords.size() + "] records to load using update"); - if(CollectionUtils.nullSafeIsEmpty(inputRecords)) - { - runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0); - return; - } - - ///////////////////////////////////////////////////////////////// - // put the destination table name in all records being updated // - ///////////////////////////////////////////////////////////////// - String table = runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE); - for(QRecord record : inputRecords) - { - record.setTableName(table); - } - - ////////////////////////////////////////// - // run an update request on the records // - ////////////////////////////////////////// - int recordsUpdated = 0; - List outputRecords = new ArrayList<>(); - int pageSize = 1000; // todo - make this a field? - - for(List page : CollectionUtils.getPages(inputRecords, pageSize)) - { - LOG.info("Updating a page of [" + page.size() + "] records. Progress: " + recordsUpdated + " loaded out of " + inputRecords.size() + " total"); - runBackendStepInput.getAsyncJobCallback().updateStatus("Updating records", recordsUpdated, inputRecords.size()); - - UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); - updateInput.setSession(runBackendStepInput.getSession()); - updateInput.setTableName(table); - updateInput.setRecords(page); - - UpdateAction updateAction = new UpdateAction(); - UpdateOutput updateResult = updateAction.execute(updateInput); - outputRecords.addAll(updateResult.getRecords()); - - recordsUpdated += updateResult.getRecords().size(); - } - runBackendStepOutput.setRecords(outputRecords); - runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsUpdated); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index 0e28a20a..b9c976f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -62,6 +62,8 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe try { + runBackendStepInput.getAsyncJobCallback().updateStatus("Executing Process"); + /////////////////////////////////////////////////////// // set up the extract, transform, and load functions // /////////////////////////////////////////////////////// @@ -137,6 +139,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe ////////////////////////////////////////////////////////////////////////////// // set the flag to state that the basepull timestamp should be updated now. // + // (upstream will check if the process was running as a basepull) // ////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue(RunProcessAction.BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD, true); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index e7f6d16e..b566c817 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -32,7 +32,11 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -54,6 +58,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { Integer limit = PROCESS_OUTPUT_RECORD_LIST_LIMIT; // todo - use a field instead of hard-coded here? + runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Preview"); ////////////////////////////////////////////////////////////////////////////////////////////////////////// // if the do-full-validation flag has already been set, then do the validation step instead of this one // @@ -69,7 +74,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe if(runBackendStepInput.getFrontendStepBehavior() != null && runBackendStepInput.getFrontendStepBehavior().equals(RunProcessInput.FrontendStepBehavior.SKIP)) { - LOG.debug("Skipping preview because frontent behavior is [" + RunProcessInput.FrontendStepBehavior.SKIP + "]."); + LOG.debug("Skipping preview because frontend behavior is [" + RunProcessInput.FrontendStepBehavior.SKIP + "]."); return; } @@ -91,8 +96,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe extractStep.setRecordPipe(recordPipe); extractStep.preRun(runBackendStepInput, runBackendStepOutput); - Integer recordCount = extractStep.doCount(runBackendStepInput); - runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + countRecords(runBackendStepInput, runBackendStepOutput, extractStep); AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); transformStep.preRun(runBackendStepInput, runBackendStepOutput); @@ -126,6 +130,26 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe + /******************************************************************************* + ** + *******************************************************************************/ + private void countRecords(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, AbstractExtractStep extractStep) throws QException + { + String sourceTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE); + QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(sourceTableName); + if(StringUtils.hasContent(sourceTableName)) + { + QBackendMetaData sourceTableBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTableName); + if(sourceTable.isCapabilityEnabled(sourceTableBackend, Capability.TABLE_COUNT)) + { + Integer recordCount = extractStep.doCount(runBackendStepInput); + runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java index ac9b9e8c..c0b9612d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -81,6 +81,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back ////////////////////////////////////////////////////////// // basically repeat the preview step, but with no limit // ////////////////////////////////////////////////////////// + runBackendStepInput.getAsyncJobCallback().updateStatus("Validating Records"); RecordPipe recordPipe = new RecordPipe(); AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); extractStep.setLimit(null); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 3f96d373..afab064d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; /******************************************************************************* @@ -389,5 +390,17 @@ public class StreamedETLWithFrontendProcess return (this); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withBasepullConfiguration(BasepullConfiguration basepullConfiguration) + { + processMetaData.setBasepullConfiguration(basepullConfiguration); + return (this); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java index 31aa8744..6b2607a3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java @@ -65,6 +65,7 @@ public class MockBackendStep implements BackendStep runBackendStepOutput.setValues(runBackendStepInput.getValues()); runBackendStepOutput.addValue(FIELD_MOCK_VALUE, MOCK_VALUE); runBackendStepOutput.addValue("noOfPeopleGreeted", runBackendStepInput.getRecords().size()); + runBackendStepOutput.addValue(RunProcessAction.BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD, true); runBackendStepOutput.addValue(RunProcessAction.BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD, true); if("there".equalsIgnoreCase(runBackendStepInput.getValueString(FIELD_GREETING_SUFFIX))) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 335956c3..e9543bbd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablesync; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -45,11 +44,15 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer; -import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils; +import com.kingsrook.qqq.backend.core.processes.utils.RecordLookupHelper; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -69,44 +72,11 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt .withSingularPastMessage("was not synced, because it is ") .withPluralPastMessage("were not synced, because they are "); - private RunBackendStepInput runBackendStepInput = null; + protected RunBackendStepInput runBackendStepInput = null; + protected RecordLookupHelper recordLookupHelper = null; private QPossibleValueTranslator possibleValueTranslator; - private Map> tableMaps = new HashMap<>(); - - - - /******************************************************************************* - ** - *******************************************************************************/ - protected QRecord getRecord(String tableName, String fieldName, Serializable value) throws QException - { - if(!tableMaps.containsKey(tableName)) - { - Map recordMap = GeneralProcessUtils.loadTableToMap(runBackendStepInput, tableName, fieldName); - tableMaps.put(tableName, recordMap); - } - - return (tableMaps.get(tableName).get(value)); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - protected Serializable getRecordField(String tableName, String fieldName, Serializable value, String outputField) throws QException - { - QRecord record = getRecord(tableName, fieldName, value); - if(record == null) - { - return (null); - } - - return (record.getValue(outputField)); - } - /******************************************************************************* @@ -121,12 +91,65 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt /******************************************************************************* + ** Map a record from the source table to the destination table. e.g., put + ** values into the destinationRecord, from the sourceRecord. ** + ** The destinationRecord will already be constructed, and will actually already + ** be the record being updated, in the case of an update. It'll be empty (newly + ** constructed) for an insert. *******************************************************************************/ public abstract QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException; + /******************************************************************************* + ** Specify a list of tableName/keyColumnName pairs to run through + ** the preloadRecords method of the recordLookupHelper. + *******************************************************************************/ + protected List> getLookupsToPreLoad() + { + return (null); + } + + + + /******************************************************************************* + ** Define the query filter to find existing records. e.g., for determining + ** insert vs. update. Subclasses may override this to customize the behavior, + ** e.g., in case an additional field is needed in the query. + *******************************************************************************/ + protected QQueryFilter getExistingRecordQueryFilter(RunBackendStepInput runBackendStepInput, List sourceKeyList) + { + String destinationTableForeignKeyField = getSyncProcessConfig().destinationTableForeignKey; + return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList)); + } + + + + /******************************************************************************* + ** Define the config for this process - e.g., what fields & tables are used. + *******************************************************************************/ + protected abstract SyncProcessConfig getSyncProcessConfig(); + + + + /******************************************************************************* + ** Record to store the config for this process - e.g., what fields & tables are used. + *******************************************************************************/ + public record SyncProcessConfig(String sourceTable, String sourceTableKeyField, String destinationTable, String destinationTableForeignKey) + { + /******************************************************************************* + ** artificial method, here to make jacoco see that this class is indeed + ** included in test coverage... + *******************************************************************************/ + void noop() + { + System.out.println("noop"); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -139,9 +162,33 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt } this.runBackendStepInput = runBackendStepInput; - String sourceTableKeyField = runBackendStepInput.getValueString(TableSyncProcess.FIELD_SOURCE_TABLE_KEY_FIELD); - String destinationTableForeignKeyField = runBackendStepInput.getValueString(TableSyncProcess.FIELD_DESTINATION_TABLE_FOREIGN_KEY); - String destinationTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE); + + if(this.recordLookupHelper == null) + { + initializeRecordLookupHelper(runBackendStepInput); + } + + SyncProcessConfig config = getSyncProcessConfig(); + + String sourceTableKeyField = config.sourceTableKeyField; + String destinationTableForeignKeyField = config.destinationTableForeignKey; + String destinationTableName = config.destinationTable; + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, destinationTableName); + + if(!StringUtils.hasContent(sourceTableKeyField)) + { + throw (new IllegalStateException("Missing sourceTableKeyField in config for " + getClass().getSimpleName())); + } + + if(!StringUtils.hasContent(destinationTableForeignKeyField)) + { + throw (new IllegalStateException("Missing destinationTableForeignKey in config for " + getClass().getSimpleName())); + } + + if(!StringUtils.hasContent(destinationTableName)) + { + throw (new IllegalStateException("Missing destinationTable in config for " + getClass().getSimpleName())); + } ////////////////////////////////////// // extract keys from source records // @@ -161,9 +208,8 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance()); queryInput.setSession(runBackendStepInput.getSession()); queryInput.setTableName(destinationTableName); - queryInput.setFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList)) - ); + QQueryFilter filter = getExistingRecordQueryFilter(runBackendStepInput, sourceKeyList); + queryInput.setFilter(filter); QueryOutput queryOutput = new QueryAction().execute(queryInput); existingRecordsByForeignKey = CollectionUtils.recordsToMap(queryOutput.getRecords(), destinationTableForeignKeyField); } @@ -171,10 +217,10 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt ///////////////////////////////////////////////////////////////// // foreach source record, build the record we'll insert/update // ///////////////////////////////////////////////////////////////// + QFieldMetaData destinationForeignKeyField = runBackendStepInput.getInstance().getTable(destinationTableName).getField(destinationTableForeignKeyField); for(QRecord sourceRecord : runBackendStepInput.getRecords()) { Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField); - QRecord existingRecord = existingRecordsByForeignKey.get(sourceKeyValue); if(sourceKeyValue == null || "".equals(sourceKeyValue)) { @@ -194,6 +240,14 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt continue; } + ///////////////////////////////////////////////////////////////////////////////////////////////// + // look for the existing record - note - we may need to type-convert here, the sourceKey value // + // from the source table to the destinationKey. e.g., if source table had an integer, and the // + // destination has a string. // + ///////////////////////////////////////////////////////////////////////////////////////////////// + Serializable sourceKeyValueInTargetFieldType = ValueUtils.getValueAsFieldType(destinationForeignKeyField.getType(), sourceKeyValue); + QRecord existingRecord = existingRecordsByForeignKey.get(sourceKeyValueInTargetFieldType); + QRecord recordToStore; if(existingRecord != null) { @@ -227,4 +281,25 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt } } + + + /******************************************************************************* + ** If needed, init a record lookup helper for this process. + *******************************************************************************/ + protected void initializeRecordLookupHelper(RunBackendStepInput runBackendStepInput) throws QException + { + this.recordLookupHelper = new RecordLookupHelper(runBackendStepInput); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there's only 1 record, don't bother preloading all records - just do the single lookup by the single needed key. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepInput.getRecords().size() > 1) + { + for(Pair pair : CollectionUtils.nonNullList(getLookupsToPreLoad())) + { + recordLookupHelper.preloadRecords(pair.getA(), pair.getB()); + } + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java index 35c6224a..fff49bc1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java @@ -24,12 +24,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablesync; import java.util.Collections; import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertOrUpdateStep; @@ -84,60 +86,28 @@ public class TableSyncProcess /******************************************************************************* - ** Fluent setter for sourceTableKeyField - ** - *******************************************************************************/ - public Builder withSourceTableKeyField(String sourceTableKeyField) - { - setInputFieldDefaultValue(FIELD_SOURCE_TABLE_KEY_FIELD, sourceTableKeyField); - return (this); - } - - - - /******************************************************************************* - ** Fluent setter for destinationTableForeignKeyField - ** - *******************************************************************************/ - public Builder withDestinationTableForeignKeyField(String destinationTableForeignKeyField) - { - setInputFieldDefaultValue(FIELD_DESTINATION_TABLE_FOREIGN_KEY, destinationTableForeignKeyField); - return (this); - } - - - - /******************************************************************************* - ** Fluent setter for transformStepClass + ** Fluent setter for transformStepClass. Note - call this method also makes + ** sourceTable and destinationTable be set - by getting them from the + ** SyncProcessConfig record defined in the step class. ** *******************************************************************************/ public Builder withSyncTransformStepClass(Class transformStepClass) { setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE, new QCodeReference(transformStepClass)); - return (this); - } + AbstractTableSyncTransformStep.SyncProcessConfig config; + try + { + AbstractTableSyncTransformStep transformStep = transformStepClass.getConstructor().newInstance(); + config = transformStep.getSyncProcessConfig(); + } + catch(Exception e) + { + throw (new QRuntimeException("Error setting up process with transform step class: " + transformStepClass.getName(), e)); + } - - /******************************************************************************* - ** Fluent setter for sourceTable - ** - *******************************************************************************/ - public Builder withSourceTable(String sourceTable) - { - setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, sourceTable); - return (this); - } - - - - /******************************************************************************* - ** Fluent setter for destinationTable - ** - *******************************************************************************/ - public Builder withDestinationTable(String destinationTable) - { - setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, destinationTable); + setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, config.sourceTable()); + setInputFieldDefaultValue(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, config.destinationTable()); return (this); } @@ -204,5 +174,17 @@ public class TableSyncProcess return (this); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public StreamedETLWithFrontendProcess.Builder withBasepullConfiguration(BasepullConfiguration basepullConfiguration) + { + processMetaData.setBasepullConfiguration(basepullConfiguration); + return (this); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java index e1b7e137..59bfe44a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java @@ -203,10 +203,15 @@ public class GeneralProcessUtils *******************************************************************************/ public static Optional getRecordByField(AbstractActionInput parentActionInput, String tableName, String fieldName, Serializable fieldValue) throws QException { + if(fieldValue == null) + { + return (Optional.empty()); + } + QueryInput queryInput = new QueryInput(parentActionInput.getInstance()); queryInput.setSession(parentActionInput.getSession()); queryInput.setTableName(tableName); - queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, List.of(fieldValue)))); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue))); queryInput.setLimit(1); QueryOutput queryOutput = new QueryAction().execute(queryInput); return (queryOutput.getRecords().stream().findFirst()); @@ -321,10 +326,26 @@ public class GeneralProcessUtils ** too many rows... Caveat emptor. *******************************************************************************/ public static Map loadTableToMap(AbstractActionInput parentActionInput, String tableName, String keyFieldName) throws QException + { + return (loadTableToMap(parentActionInput, tableName, keyFieldName, (QQueryFilter) null)); + } + + + + /******************************************************************************* + ** Load rows from a table matching the specified filter, into a map, keyed by the keyFieldName. + ** + ** Note - null values from the key field are NOT put in the map. + ** + ** If multiple values are found for the key, they'll squash each other, and only + ** one (random) value will appear. + *******************************************************************************/ + public static Map loadTableToMap(AbstractActionInput parentActionInput, String tableName, String keyFieldName, QQueryFilter filter) throws QException { QueryInput queryInput = new QueryInput(parentActionInput.getInstance()); queryInput.setSession(parentActionInput.getSession()); queryInput.setTableName(tableName); + queryInput.setFilter(filter); QueryOutput queryOutput = new QueryAction().execute(queryInput); List records = queryOutput.getRecords(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java new file mode 100644 index 00000000..a09d6bef --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java @@ -0,0 +1,153 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.utils; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Utility to help processes lookup records. Caches lookups - and potentially + ** can pre-load entire tables or subsets of tables. + ** + *******************************************************************************/ +public class RecordLookupHelper +{ + private final AbstractActionInput actionInput; + + private Map> foreignRecordMaps = new HashMap<>(); + private Set preloadedKeys = new HashSet<>(); + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RecordLookupHelper(AbstractActionInput actionInput) + { + this.actionInput = actionInput; + } + + + + /******************************************************************************* + ** Fetch a record from a table by a key field (doesn't have to be its primary key). + *******************************************************************************/ + public QRecord getRecordByKey(String tableName, String keyFieldName, Serializable key) throws QException + { + String mapKey = tableName + "." + keyFieldName; + Map recordMap = foreignRecordMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>()); + + if(!recordMap.containsKey(key)) + { + Optional optRecord = GeneralProcessUtils.getRecordByField(actionInput, tableName, keyFieldName, key); + recordMap.put(key, optRecord.orElse(null)); + } + + return (recordMap.get(key)); + } + + + + /******************************************************************************* + ** Optimization - to pre-load the records in a single query, which would otherwise + ** have to be looked up one-by-one. + *******************************************************************************/ + public void preloadRecords(String tableName, String keyFieldName) throws QException + { + String mapKey = tableName + "." + keyFieldName; + if(!preloadedKeys.contains(mapKey)) + { + Map recordMap = GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName); + foreignRecordMaps.put(mapKey, recordMap); + preloadedKeys.add(mapKey); + } + } + + + + /******************************************************************************* + ** Get a value from a record, by doing a lookup on the specified keyFieldName, + ** for the specified key value. + ** + *******************************************************************************/ + public Serializable getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key) throws QException + { + QRecord record = getRecordByKey(tableName, keyFieldName, key); + if(record == null) + { + return (null); + } + + return (record.getValue(requestedField)); + } + + + + /******************************************************************************* + ** Get a value from a record, in the requested type, by doing a lookup on the + ** specified keyFieldName, for the specified key value. + ** + *******************************************************************************/ + public T getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key, Class type) throws QException + { + Serializable value = getRecordValue(tableName, requestedField, keyFieldName, key); + return (ValueUtils.getValueAsType(type, value)); + } + + + + /******************************************************************************* + ** Get the id (primary key) value from a record, by doing a lookup on the + ** specified keyFieldName, for the specified key value. + ** + *******************************************************************************/ + public Serializable getRecordId(String tableName, String keyFieldName, Serializable key) throws QException + { + String primaryKeyField = actionInput.getInstance().getTable(tableName).getPrimaryKeyField(); + return (getRecordValue(tableName, primaryKeyField, keyFieldName, key)); + } + + + + /******************************************************************************* + ** Get the id (primary key) value from a record, in the requested type, by doing + ** a lookup on the specified keyFieldName, for the specified key value. + ** + *******************************************************************************/ + public T getRecordId(String tableName, String keyFieldName, Serializable key, Class type) throws QException + { + Serializable value = getRecordId(tableName, keyFieldName, key); + return (ValueUtils.getValueAsType(type, value)); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/Pair.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/Pair.java index 3fc17e48..086b6d52 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/Pair.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/Pair.java @@ -46,6 +46,16 @@ public class Pair implements Cloneable + /******************************************************************************* + ** static constructor (factory) + *******************************************************************************/ + public static Pair of(A a, B b) + { + return (new Pair<>(a, b)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java index cea36216..ba66dcbb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -53,7 +54,8 @@ public class QPossibleValueTranslatorTest ** *******************************************************************************/ @BeforeEach - void beforeEach() + @AfterEach + void beforeAndAfterEach() { MemoryRecordStore.getInstance().reset(); MemoryRecordStore.resetStatistics(); @@ -164,6 +166,7 @@ public class QPossibleValueTranslatorTest /////////////////////////////////////////////////////////// possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession()); MemoryRecordStore.setCollectStatistics(true); + MemoryRecordStore.resetStatistics(); possibleValueTranslator.translatePossibleValue(shapeField, 1); possibleValueTranslator.translatePossibleValue(shapeField, 2); assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 queries so far"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Shape.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Shape.java new file mode 100644 index 00000000..b9427ce4 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Shape.java @@ -0,0 +1,316 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.data.testentities; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class Shape extends QRecordEntity +{ + @QField() + private Integer id; + + @QField() + private Instant createDate; + + @QField() + private Instant modifyDate; + + @QField() + private String name; + + @QField() + private String type; + + @QField() + private Integer noOfSides; + + @QField() + private Boolean isPolygon; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Shape() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Shape(QRecord record) + { + populateFromQRecord(record); + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Setter for id + ** + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + ** + *******************************************************************************/ + public Shape withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public Shape withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + ** + *******************************************************************************/ + public Instant getCreateDate() + { + return createDate; + } + + + + /******************************************************************************* + ** Setter for createDate + ** + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + ** + *******************************************************************************/ + public Shape withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + ** + *******************************************************************************/ + public Instant getModifyDate() + { + return modifyDate; + } + + + + /******************************************************************************* + ** Setter for modifyDate + ** + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + ** + *******************************************************************************/ + public Shape withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public String getType() + { + return type; + } + + + + /******************************************************************************* + ** Setter for type + ** + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + ** + *******************************************************************************/ + public Shape withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** Getter for noOfSides + ** + *******************************************************************************/ + public Integer getNoOfSides() + { + return noOfSides; + } + + + + /******************************************************************************* + ** Setter for noOfSides + ** + *******************************************************************************/ + public void setNoOfSides(Integer noOfSides) + { + this.noOfSides = noOfSides; + } + + + + /******************************************************************************* + ** Fluent setter for noOfSides + ** + *******************************************************************************/ + public Shape withNoOfSides(Integer noOfSides) + { + this.noOfSides = noOfSides; + return (this); + } + + + + /******************************************************************************* + ** Getter for isPolygon + ** + *******************************************************************************/ + public Boolean getIsPolygon() + { + return isPolygon; + } + + + + /******************************************************************************* + ** Setter for isPolygon + ** + *******************************************************************************/ + public void setIsPolygon(Boolean isPolygon) + { + this.isPolygon = isPolygon; + } + + + + /******************************************************************************* + ** Fluent setter for isPolygon + ** + *******************************************************************************/ + public Shape withIsPolygon(Boolean isPolygon) + { + this.isPolygon = isPolygon; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java index caaa28e9..e7ed7ea8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java @@ -55,7 +55,7 @@ class TableSyncProcessTest ** *******************************************************************************/ @Test - void test() throws QException + void test() throws Exception { QInstance qInstance = TestUtils.defineInstance(); @@ -86,10 +86,6 @@ class TableSyncProcessTest qInstance.addProcess(TableSyncProcess.processMetaDataBuilder(false) .withName(PROCESS_NAME) .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) - .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) - .withDestinationTable(TABLE_NAME_PEOPLE_SYNC) - .withSourceTableKeyField("id") - .withDestinationTableForeignKeyField("sourcePersonId") .withSyncTransformStepClass(PersonTransformClass.class) .getProcessMetaData()); @@ -142,6 +138,16 @@ class TableSyncProcessTest return (destinationRecord); } + + + @Override + protected SyncProcessConfig getSyncProcessConfig() + { + SyncProcessConfig syncProcessConfig = new SyncProcessConfig(TestUtils.TABLE_NAME_PERSON_MEMORY, "id", "peopleSync", "sourcePersonId"); + syncProcessConfig.noop(); + return (syncProcessConfig); + } + } } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtilsTest.java index 433f5f96..cd0d44f2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtilsTest.java @@ -30,12 +30,14 @@ import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.testentities.Shape; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; @@ -46,7 +48,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -329,6 +333,11 @@ class GeneralProcessUtilsTest assertEquals(3, recordMapByFirstName.size()); assertEquals(1, recordMapByFirstName.get("Darin").getValueInteger("id")); assertEquals(3, recordMapByFirstName.get("Tim").getValueInteger("id")); + + Map recordMapByFirstNameAsString = GeneralProcessUtils.loadTableToMap(queryInput, TestUtils.TABLE_NAME_PERSON_MEMORY, String.class, "firstName"); + assertEquals(3, recordMapByFirstName.size()); + assertEquals(1, recordMapByFirstName.get("Darin").getValueInteger("id")); + assertEquals(3, recordMapByFirstName.get("Tim").getValueInteger("id")); } @@ -354,4 +363,123 @@ class GeneralProcessUtilsTest assertEquals(1, map.get("Darin").size()); assertEquals(2, map.get("James").size()); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetRecordByFieldOrElseThrow() throws QException + { + QInstance instance = TestUtils.defineInstance(); + TestUtils.insertDefaultShapes(instance); + + assertNotNull(GeneralProcessUtils.getRecordByFieldOrElseThrow(new AbstractActionInput(instance, new QSession()), TestUtils.TABLE_NAME_SHAPE, "name", "Triangle")); + assertThrows(QException.class, () -> GeneralProcessUtils.getRecordByFieldOrElseThrow(new AbstractActionInput(instance, new QSession()), TestUtils.TABLE_NAME_SHAPE, "name", "notAShape")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetRecordByPrimaryKey() throws QException + { + QInstance instance = TestUtils.defineInstance(); + TestUtils.insertDefaultShapes(instance); + + AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession()); + assertTrue(GeneralProcessUtils.getRecordByPrimaryKey(actionInput, TestUtils.TABLE_NAME_SHAPE, 1).isPresent()); + assertFalse(GeneralProcessUtils.getRecordByPrimaryKey(actionInput, TestUtils.TABLE_NAME_SHAPE, -1).isPresent()); + assertNotNull(GeneralProcessUtils.getRecordByPrimaryKeyOrElseThrow(actionInput, TestUtils.TABLE_NAME_SHAPE, 1)); + assertThrows(QException.class, () -> GeneralProcessUtils.getRecordByPrimaryKeyOrElseThrow(actionInput, TestUtils.TABLE_NAME_SHAPE, -1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCount() throws QException + { + QInstance instance = TestUtils.defineInstance(); + TestUtils.insertDefaultShapes(instance); + AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession()); + + assertEquals(3, GeneralProcessUtils.count(actionInput, TestUtils.TABLE_NAME_SHAPE, null)); + assertEquals(1, GeneralProcessUtils.count(actionInput, TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 2)))); + assertEquals(0, GeneralProcessUtils.count(actionInput, TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IS_BLANK)))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetEntityByField() throws QException + { + QInstance instance = TestUtils.defineInstance(); + TestUtils.insertDefaultShapes(instance); + AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession()); + + assertEquals("Triangle", GeneralProcessUtils.getEntityByField(actionInput, TestUtils.TABLE_NAME_SHAPE, "name", "Triangle", Shape.class).get().getName()); + assertFalse(GeneralProcessUtils.getEntityByField(actionInput, TestUtils.TABLE_NAME_SHAPE, "name", "notAShape", Shape.class).isPresent()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLoadTableAsEntities() throws QException + { + QInstance instance = TestUtils.defineInstance(); + TestUtils.insertDefaultShapes(instance); + AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession()); + + List shapes = GeneralProcessUtils.loadTable(actionInput, TestUtils.TABLE_NAME_SHAPE, Shape.class); + assertEquals(3, shapes.size()); + assertTrue(shapes.get(0) instanceof Shape); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLoadTableToMapAsEntities() throws QException + { + QInstance instance = TestUtils.defineInstance(); + TestUtils.insertDefaultShapes(instance); + AbstractActionInput actionInput = new AbstractActionInput(instance, new QSession()); + + Map map = GeneralProcessUtils.loadTableToMap(actionInput, TestUtils.TABLE_NAME_SHAPE, "id", Shape.class); + assertEquals(3, map.size()); + assertTrue(map.get(1) instanceof Shape); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordsToEntities() throws QException + { + List shapes = GeneralProcessUtils.recordsToEntities(Shape.class, List.of( + new QRecord().withValue("id", 99).withValue("name", "round"), + new QRecord().withValue("id", 98).withValue("name", "flat") + )); + + assertEquals(2, shapes.size()); + assertEquals(99, shapes.get(0).getId()); + assertEquals("round", shapes.get(0).getName()); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java new file mode 100644 index 00000000..73f9a42c --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.utils; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for RecordLookupHelper + *******************************************************************************/ +class RecordLookupHelperTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + MemoryRecordStore.resetStatistics(); + MemoryRecordStore.setCollectStatistics(true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithoutPreload() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + TestUtils.insertDefaultShapes(qInstance); + RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput(qInstance, new QSession())); + + MemoryRecordStore.setCollectStatistics(true); + assertEquals(2, recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "Square")); + assertEquals(2, recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "Square", Integer.class)); + assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + + assertEquals("Circle", recordLookupHelper.getRecordValue(TestUtils.TABLE_NAME_SHAPE, "name", "id", 3)); + assertEquals("Circle", recordLookupHelper.getRecordValue(TestUtils.TABLE_NAME_SHAPE, "name", "id", 3, String.class)); + assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + + assertNull(recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "notAShape")); + assertNull(recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "notAShape", Integer.class)); + assertEquals(3, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithPreload() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + TestUtils.insertDefaultShapes(qInstance); + + RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput(qInstance, new QSession())); + recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name"); + assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + + assertNotNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Triangle")); + assertEquals(1, recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Triangle").getValueInteger("id")); + assertEquals("Triangle", recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Triangle").getValueString("name")); + assertEquals(2, recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Square").getValueInteger("id")); + assertEquals("Square", recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Square").getValueString("name")); + assertEquals(3, recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Circle").getValueInteger("id")); + assertEquals("Circle", recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Circle").getValueString("name")); + + ///////////////////////////////////////////////////// + // all those gets should run no additional queries // + ///////////////////////////////////////////////////// + assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + + //////////////////////////////////////////////////////////////////// + // make sure we don't re-do the query in a second call to preload // + //////////////////////////////////////////////////////////////////// + recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name"); + assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + + /////////////////////////////////////////////////// + // make sure we can preload by a different field // + /////////////////////////////////////////////////// + recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "id"); + assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 2a6a3b1a..7de6d174 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -193,9 +193,17 @@ public abstract class AbstractRDBMSAction implements QActionInterface //////////////////////////////////////////////////////////// // find the join in the instance, to see the 'on' clause // //////////////////////////////////////////////////////////// - List joinClauseList = new ArrayList<>(); - String leftTableName = joinsContext.resolveTableNameOrAliasToTableName(queryJoin.getLeftTableOrAlias()); - QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> findJoinMetaData(instance, leftTableName, queryJoin.getRightTable())); + List joinClauseList = new ArrayList<>(); + String leftTableName = joinsContext.resolveTableNameOrAliasToTableName(queryJoin.getLeftTableOrAlias()); + QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> + { + QJoinMetaData found = findJoinMetaData(instance, leftTableName, queryJoin.getRightTable()); + if(found == null) + { + throw (new RuntimeException("Could not find a join between tables [" + leftTableName + "][" + queryJoin.getRightTable() + "]")); + } + return (found); + }); for(JoinOn joinOn : joinMetaData.getJoinOns()) { QTableMetaData leftTable = instance.getTable(joinMetaData.getLeftTable()); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index fb1c32dc..92acbc89 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -184,6 +184,7 @@ public class QJavalinImplementation service = Javalin.create().start(port); service.routes(getRoutes()); service.before(QJavalinImplementation::hotSwapQInstance); + service.before((Context context) -> context.header("Content-Type", "application/json")); } @@ -815,26 +816,9 @@ public class QJavalinImplementation String filter = context.queryParam("filter"); Integer limit = integerQueryParam(context, "limit"); - ///////////////////////////////////////////////////////////////////////////////////////// - // if a format query param wasn't given, then try to get file extension from file name // - ///////////////////////////////////////////////////////////////////////////////////////// - if(!StringUtils.hasContent(format) && optionalFilename.isPresent() && StringUtils.hasContent(optionalFilename.get())) + ReportFormat reportFormat = getReportFormat(context, optionalFilename, format); + if(reportFormat == null) { - String filename = optionalFilename.get(); - if(filename.contains(".")) - { - format = filename.substring(filename.lastIndexOf(".") + 1); - } - } - - ReportFormat reportFormat; - try - { - reportFormat = ReportFormat.fromString(format); - } - catch(QUserFacingException e) - { - handleException(HttpStatus.Code.BAD_REQUEST, context, e); return; } @@ -861,55 +845,21 @@ public class QJavalinImplementation exportInput.setQueryFilter(JsonUtils.toObject(filter, QQueryFilter.class)); } - /////////////////////////////////////////////////////////////////////////////////////////////////////// - // set up the I/O pipe streams. // - // Critically, we must NOT open the outputStream in a try-with-resources. The thread that writes to // - // the stream must close it when it's done writing. // - /////////////////////////////////////////////////////////////////////////////////////////////////////// - PipedOutputStream pipedOutputStream = new PipedOutputStream(); - PipedInputStream pipedInputStream = new PipedInputStream(); - pipedOutputStream.connect(pipedInputStream); - exportInput.setReportOutputStream(pipedOutputStream); - - ExportAction exportAction = new ExportAction(); - exportAction.preExecute(exportInput); - - ///////////////////////////////////////////////////////////////////////////////////////////////////// - // start the async job. // - // Critically, this must happen before the pipedInputStream is passed to the javalin result method // - ///////////////////////////////////////////////////////////////////////////////////////////////////// - new AsyncJobManager().startJob("Javalin>ReportAction", (o) -> + UnsafeFunction preAction = (PipedOutputStream pos) -> { - try - { - exportAction.execute(exportInput); - return (true); - } - catch(Exception e) - { - pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes()); - pipedOutputStream.close(); - return (false); - } - }); + exportInput.setReportOutputStream(pos); - //////////////////////////////////////////// - // set the response content type & stream // - //////////////////////////////////////////// - context.contentType(reportFormat.getMimeType()); - context.header("Content-Disposition", "filename=" + filename); - context.result(pipedInputStream); + ExportAction exportAction = new ExportAction(); + exportAction.preExecute(exportInput); + return (exportAction); + }; - //////////////////////////////////////////////////////////////////////////////////////////// - // we'd like to check to see if the job failed, and if so, to give the user an error... // - // but if we "block" here, then piped streams seem to never flush, so we deadlock things. // - //////////////////////////////////////////////////////////////////////////////////////////// - // AsyncJobStatus asyncJobStatus = asyncJobManager.waitForJob(jobUUID); - // if(asyncJobStatus.getState().equals(AsyncJobState.ERROR)) - // { - // System.out.println("Well, here we are..."); - // throw (new QUserFacingException("Error running report: " + asyncJobStatus.getCaughtException().getMessage())); - // } + UnsafeConsumer execute = (ExportAction exportAction) -> + { + exportAction.execute(exportInput); + }; + + runStreamedExportOrReport(context, reportFormat, filename, preAction, execute); } catch(Exception e) { @@ -919,6 +869,122 @@ public class QJavalinImplementation + /******************************************************************************* + ** + *******************************************************************************/ + @FunctionalInterface + public interface UnsafeFunction + { + /******************************************************************************* + ** + *******************************************************************************/ + R run(T t) throws Exception; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @FunctionalInterface + public interface UnsafeConsumer + { + /******************************************************************************* + ** + *******************************************************************************/ + void run(T t) throws Exception; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void runStreamedExportOrReport(Context context, ReportFormat reportFormat, String filename, UnsafeFunction preAction, UnsafeConsumer executor) throws Exception + { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up the I/O pipe streams. // + // Critically, we must NOT open the outputStream in a try-with-resources. The thread that writes to // + // the stream must close it when it's done writing. // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + PipedOutputStream pipedOutputStream = new PipedOutputStream(); + PipedInputStream pipedInputStream = new PipedInputStream(); + pipedOutputStream.connect(pipedInputStream); + + T t = preAction.run(pipedOutputStream); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // start the async job. // + // Critically, this must happen before the pipedInputStream is passed to the javalin result method // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + new AsyncJobManager().startJob("Javalin>ExportAction", (o) -> + { + try + { + executor.run(t); + return (true); + } + catch(Exception e) + { + pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes()); + pipedOutputStream.close(); + return (false); + } + }); + + //////////////////////////////////////////// + // set the response content type & stream // + //////////////////////////////////////////// + context.contentType(reportFormat.getMimeType()); + context.header("Content-Disposition", "filename=" + filename); + context.result(pipedInputStream); + + //////////////////////////////////////////////////////////////////////////////////////////// + // we'd like to check to see if the job failed, and if so, to give the user an error... // + // but if we "block" here, then piped streams seem to never flush, so we deadlock things. // + //////////////////////////////////////////////////////////////////////////////////////////// + // AsyncJobStatus asyncJobStatus = asyncJobManager.waitForJob(jobUUID); + // if(asyncJobStatus.getState().equals(AsyncJobState.ERROR)) + // { + // System.out.println("Well, here we are..."); + // throw (new QUserFacingException("Error running report: " + asyncJobStatus.getCaughtException().getMessage())); + // } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ReportFormat getReportFormat(Context context, Optional optionalFilename, String format) + { + ///////////////////////////////////////////////////////////////////////////////////////// + // if a format query param wasn't given, then try to get file extension from file name // + ///////////////////////////////////////////////////////////////////////////////////////// + if(!StringUtils.hasContent(format) && optionalFilename.isPresent() && StringUtils.hasContent(optionalFilename.get())) + { + String filename = optionalFilename.get(); + if(filename.contains(".")) + { + format = filename.substring(filename.lastIndexOf(".") + 1); + } + } + + ReportFormat reportFormat; + try + { + reportFormat = ReportFormat.fromString(format); + } + catch(QUserFacingException e) + { + handleException(HttpStatus.Code.BAD_REQUEST, context, e); + return null; + } + return reportFormat; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -997,21 +1063,21 @@ public class QJavalinImplementation { if(userFacingException instanceof QNotFoundException) { - int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.NOT_FOUND).getCode(); - context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}"); + statusCode = Objects.requireNonNullElse(statusCode, HttpStatus.Code.NOT_FOUND); // 404 + respondWithError(context, statusCode, userFacingException.getMessage()); } else { LOG.info("User-facing exception", e); - int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode(); - context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}"); + statusCode = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR); // 500 + respondWithError(context, statusCode, userFacingException.getMessage()); } } else { if(e instanceof QAuthenticationException) { - context.status(HttpStatus.UNAUTHORIZED_401).result("{\"error\":\"" + e.getMessage() + "\"}"); + respondWithError(context, HttpStatus.Code.UNAUTHORIZED, e.getMessage()); // 401 return; } @@ -1019,13 +1085,23 @@ public class QJavalinImplementation // default exception handling // //////////////////////////////// LOG.warn("Exception in javalin request", e); - int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode(); - context.status(code).result("{\"error\":\"" + e.getClass().getSimpleName() + " (" + e.getMessage() + ")\"}"); + respondWithError(context, HttpStatus.Code.INTERNAL_SERVER_ERROR, e.getClass().getSimpleName() + " (" + e.getMessage() + ")"); // 500 } } + /******************************************************************************* + ** + *******************************************************************************/ + public static void respondWithError(Context context, HttpStatus.Code statusCode, String errorMessage) + { + context.status(statusCode.getCode()); + context.result(JsonUtils.toJson(Map.of("error", errorMessage))); + } + + + /******************************************************************************* ** Returns Integer if context has a valid int query parameter by the given name, ** Returns null if no param (or empty value). diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 9f2c9a7f..3cba884e 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -27,6 +27,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.PipedOutputStream; import java.io.Serializable; import java.time.LocalDate; import java.util.ArrayList; @@ -34,6 +35,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -45,15 +47,19 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; import com.kingsrook.qqq.backend.core.actions.async.JobGoingAsyncException; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; @@ -62,6 +68,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.state.StateType; import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; @@ -70,12 +77,14 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import io.javalin.apibuilder.EndpointGroup; import io.javalin.http.Context; import io.javalin.http.UploadedFile; import org.apache.commons.lang.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.eclipse.jetty.http.HttpStatus; import static io.javalin.apibuilder.ApiBuilder.get; import static io.javalin.apibuilder.ApiBuilder.path; import static io.javalin.apibuilder.ApiBuilder.post; @@ -115,11 +124,131 @@ public class QJavalinProcessHandler }); }); get("/download/{file}", QJavalinProcessHandler::downloadFile); + + path("/reports", () -> + { + path("/{reportName}", () -> + { + get("", QJavalinProcessHandler::reportWithoutFilename); + get("/{fileName}", QJavalinProcessHandler::reportWithFilename); + }); + }); }); } + /******************************************************************************* + ** + *******************************************************************************/ + private static void reportWithFilename(Context context) + { + String filename = context.pathParam("fileName"); + report(context, Optional.of(filename)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void reportWithoutFilename(Context context) + { + report(context, Optional.empty()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void report(Context context, Optional optionalFilename) + { + try + { + ////////////////////////////////////////// + // read params from the request context // + ////////////////////////////////////////// + String reportName = context.pathParam("reportName"); + String format = context.queryParam("format"); + + ReportFormat reportFormat = QJavalinImplementation.getReportFormat(context, optionalFilename, format); + if(reportFormat == null) + { + return; + } + + String filename = optionalFilename.orElse(reportName + "." + reportFormat.toString().toLowerCase(Locale.ROOT)); + + ///////////////////////////////////////////// + // set up the report action's input object // + ///////////////////////////////////////////// + ReportInput reportInput = new ReportInput(QJavalinImplementation.qInstance); + QJavalinImplementation.setupSession(context, reportInput); + reportInput.setReportFormat(reportFormat); + reportInput.setReportName(reportName); + reportInput.setInputValues(null); // todo! + reportInput.setFilename(filename); + + QReportMetaData report = QJavalinImplementation.qInstance.getReport(reportName); + if(report == null) + { + throw (new QNotFoundException("Report [" + reportName + "] is not found.")); + } + + ////////////////////////////////////////////////////////////// + // process the report's input fields, from the query string // + ////////////////////////////////////////////////////////////// + for(QFieldMetaData inputField : CollectionUtils.nonNullList(report.getInputFields())) + { + try + { + boolean setValue = false; + if(context.queryParamMap().containsKey(inputField.getName())) + { + String value = context.queryParamMap().get(inputField.getName()).get(0); + Serializable typedValue = ValueUtils.getValueAsFieldType(inputField.getType(), value); + reportInput.addInputValue(inputField.getName(), typedValue); + setValue = true; + } + + if(inputField.getIsRequired() && !setValue) + { + QJavalinImplementation.respondWithError(context, HttpStatus.Code.BAD_REQUEST, "Missing query param value for required input field: [" + inputField.getName() + "]"); + return; + } + } + catch(Exception e) + { + QJavalinImplementation.respondWithError(context, HttpStatus.Code.BAD_REQUEST, "Error processing query param [" + inputField.getName() + "]: " + e.getClass().getSimpleName() + " (" + e.getMessage() + ")"); + return; + } + } + + QJavalinImplementation.UnsafeFunction preAction = (PipedOutputStream pos) -> + { + reportInput.setReportOutputStream(pos); + + GenerateReportAction reportAction = new GenerateReportAction(); + // any pre-action?? export uses this for "too many rows" checks... + return (reportAction); + }; + + QJavalinImplementation.UnsafeConsumer execute = (GenerateReportAction generateReportAction) -> + { + generateReportAction.execute(reportInput); + }; + + QJavalinImplementation.runStreamedExportOrReport(context, reportFormat, filename, preAction, execute); + } + catch(Exception e) + { + QJavalinImplementation.handleException(context, e); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java index 19d971f8..f3f935c9 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java @@ -471,7 +471,7 @@ class QJavalinImplementationTest extends QJavalinTestBase { HttpResponse response = Unirest.get(BASE_URL + "/data/person/export/MyPersonExport.csv").asString(); assertEquals(200, response.getStatus()); - assertEquals("text/csv", response.getHeaders().get("Content-Type").get(0)); + assertEquals("text/csv;charset=utf-8", response.getHeaders().get("Content-Type").get(0)); assertEquals("filename=MyPersonExport.csv", response.getHeaders().get("Content-Disposition").get(0)); String[] csvLines = response.getBody().split("\n"); assertEquals(6, csvLines.length); @@ -500,7 +500,7 @@ class QJavalinImplementationTest extends QJavalinTestBase { HttpResponse response = Unirest.get(BASE_URL + "/data/person/export/?format=xlsx").asString(); assertEquals(200, response.getStatus()); - assertEquals(ReportFormat.XLSX.getMimeType(), response.getHeaders().get("Content-Type").get(0)); + assertEquals(ReportFormat.XLSX.getMimeType() + ";charset=utf-8", response.getHeaders().get("Content-Type").get(0)); assertEquals("filename=person.xlsx", response.getHeaders().get("Content-Disposition").get(0)); } diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java index 3effbf77..b53930fd 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java @@ -35,6 +35,7 @@ import kong.unirest.Unirest; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -476,4 +477,59 @@ class QJavalinProcessHandlerTest extends QJavalinTestBase getProcessRecords(processUUID, 0, 5, 5); } + + + /******************************************************************************* + ** test running a report + ** + *******************************************************************************/ + @Test + public void test_report() + { + HttpResponse response = Unirest.get(BASE_URL + "/reports/personsReport?format=csv&firstNamePrefix=D").asString(); + assertEquals(200, response.getStatus()); + assertThat(response.getHeaders().get("Content-Type").get(0)).contains("text/csv"); + assertThat(response.getHeaders().get("Content-Disposition").get(0)).contains("filename=personsReport.csv"); + String csv = response.getBody(); + System.out.println(csv); + assertThat(csv).contains(""" + "Id","First Name","Last Name\""""); + assertThat(csv).contains(""" + "1","Darin","Kelkhoff\""""); + } + + + + /******************************************************************************* + ** test running a report + ** + *******************************************************************************/ + @Test + public void test_reportMissingFormat() + { + HttpResponse response = Unirest.get(BASE_URL + "/reports/personsReport?firstNamePrefix=D").asString(); + assertEquals(400, response.getStatus()); + assertThat(response.getHeaders().get("Content-Type").get(0)).contains("application/json"); + JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertThat(jsonObject.getString("error")).contains("Report format was not specified"); + } + + + + /******************************************************************************* + ** test running a report by filename + ** + *******************************************************************************/ + @Test + public void test_reportWithFileName() + { + HttpResponse response = Unirest.get(BASE_URL + "/reports/personsReport/myFile.json?firstNamePrefix=D").asString(); + assertEquals(200, response.getStatus()); + assertThat(response.getHeaders().get("Content-Type").get(0)).contains("application/json"); + assertThat(response.getHeaders().get("Content-Disposition").get(0)).contains("filename=myFile.json"); + // JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + // System.out.println(jsonObject); + JSONArray jsonArray = JsonUtils.toJSONArray(response.getBody()); + } + } \ No newline at end of file diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java index 9a313b9d..3d9d44e6 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java @@ -30,6 +30,9 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -48,6 +51,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider; @@ -66,6 +74,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; *******************************************************************************/ public class TestUtils { + public static final String TABLE_NAME_PERSON = "person"; + public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive"; public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep"; public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow"; @@ -129,6 +139,7 @@ public class TestUtils qInstance.addProcess(defineProcessSimpleSleep()); qInstance.addProcess(defineProcessScreenThenSleep()); qInstance.addProcess(defineProcessSimpleThrow()); + qInstance.addReport(definePersonsReport()); qInstance.addPossibleValueSource(definePossibleValueSourcePerson()); defineWidgets(qInstance); @@ -210,7 +221,7 @@ public class TestUtils public static QTableMetaData defineTablePerson() { return new QTableMetaData() - .withName("person") + .withName(TABLE_NAME_PERSON) .withLabel("Person") .withRecordLabelFormat("%s %s") .withRecordLabelFields("firstName", "lastName") @@ -222,7 +233,7 @@ public class TestUtils .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) - .withField(new QFieldMetaData("partnerPersonId", QFieldType.INTEGER).withBackendName("partner_person_id").withPossibleValueSourceName("person")) + .withField(new QFieldMetaData("partnerPersonId", QFieldType.INTEGER).withBackendName("partner_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) .withField(new QFieldMetaData("email", QFieldType.STRING)) .withField(new QFieldMetaData("testScriptId", QFieldType.INTEGER).withBackendName("test_script_id")) .withAssociatedScript(new AssociatedScript() @@ -240,7 +251,7 @@ public class TestUtils { return new QProcessMetaData() .withName("greet") - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addStep(new QBackendStepMetaData() .withName("prepare") .withCode(new QCodeReference() @@ -248,14 +259,14 @@ public class TestUtils .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context? .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName("person")) + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON)) .withFieldList(List.of( new QFieldMetaData("greetingPrefix", QFieldType.STRING), new QFieldMetaData("greetingSuffix", QFieldType.STRING) ))) .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) @@ -271,7 +282,7 @@ public class TestUtils { return new QProcessMetaData() .withName(PROCESS_NAME_GREET_PEOPLE_INTERACTIVE) - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addStep(new QFrontendStepMetaData() .withName("setup") @@ -286,14 +297,14 @@ public class TestUtils .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context? .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName("person")) + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON)) .withFieldList(List.of( new QFieldMetaData("greetingPrefix", QFieldType.STRING), new QFieldMetaData("greetingSuffix", QFieldType.STRING) ))) .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) @@ -313,9 +324,9 @@ public class TestUtils private static QPossibleValueSource definePossibleValueSourcePerson() { return (new QPossibleValueSource() - .withName("person") + .withName(TABLE_NAME_PERSON) .withType(QPossibleValueSourceType.TABLE) - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_PARENS_ID) .withOrderByField("id") ); @@ -469,4 +480,28 @@ public class TestUtils } } + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportMetaData definePersonsReport() + { + return new QReportMetaData() + .withName("personsReport") + .withInputField(new QFieldMetaData("firstNamePrefix", QFieldType.STRING)) + .withDataSource(new QReportDataSource() + .withSourceTable(TABLE_NAME_PERSON) + .withQueryFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.STARTS_WITH, "${input.firstNamePrefix}"))) + ) + .withView(new QReportView() + .withType(ReportType.TABLE) + .withColumns(List.of( + new QReportField("id"), + new QReportField("firstName"), + new QReportField("lastName") + )) + ); + } + }