QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7 from Kingsrook/feature/sprint-10

Browse Source 
Feature/sprint 10
This commit is contained in:
darin.kelkhoff
2022-09-08 11:09:43 -05:00
committed by GitHub
parent c3bba7cf8c 0d0a7aec1c
commit 278910e4ee
146 changed files with 11193 additions and 1498 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
checkstyle.xml → checkstyle/config.xml
View File

@ -261,5 +261,9 @@
<module name="MissingOverride"/>
</module>
<module name="Header">
<property name="headerFile" value="checkstyle/license.txt"/>
<property name="fileExtensions" value="java"/>
</module>
<module name="SuppressWarningsFilter"/>
</module>

20
checkstyle/license.txt Normal file
View File

@ -0,0 +1,20 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

3
pom.xml
View File

@ -122,7 +122,8 @@
<id>validate</id>
<phase>validate</phase>
<configuration>
<configLocation>checkstyle.xml</configLocation>
<configLocation>checkstyle/config.xml</configLocation>
<headerLocation>checkstyle/license.txt</headerLocation>
<!-- <suppressionsLocation>checkstyle-suppressions.xml</suppressionsLocation> -->
<encoding>UTF-8</encoding>
<consoleOutput>true</consoleOutput>

15
qqq-backend-core/pom.xml
View File

@ -36,11 +36,26 @@
<!-- noe at this time -->
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>2.17.259</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<!-- other qqq modules deps -->
<!-- none, this is core. -->
<!-- 3rd party deps specifically for this module -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>quicksight</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>

25
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java
View File

@ -31,7 +31,6 @@ import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
** Argument passed to an AsyncJob when it runs, which can be used to communicate
** data back out of the job.
**
** TODO - future - allow cancellation to be indicated here?
*******************************************************************************/
public class AsyncJobCallback
{
@ -51,6 +50,17 @@ public class AsyncJobCallback
/*******************************************************************************
** Setter for asyncJobStatus
**
*******************************************************************************/
public void setAsyncJobStatus(AsyncJobStatus asyncJobStatus)
{
this.asyncJobStatus = asyncJobStatus;
}
/*******************************************************************************
** Update the message
*******************************************************************************/
@ -107,4 +117,17 @@ public class AsyncJobCallback
AsyncJobManager.getStateProvider().put(new UUIDAndTypeStateKey(jobUUID, StateType.ASYNC_JOB_STATUS), asyncJobStatus);
}
/*******************************************************************************
** Check if the asyncJobStatus had a cancellation requested.
**
** TODO - concern about multiple threads writing this object to a non-in-memory
** state provider, and this value getting lost...
*******************************************************************************/
public boolean wasCancelRequested()
{
return (this.asyncJobStatus.getCancelRequested());
}
}

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java
View File

@ -135,11 +135,11 @@ public class AsyncJobManager
Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8));
try
{
LOG.info("Starting job " + uuidAndTypeStateKey.getUuid());
LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid());
T result = asyncJob.run(new AsyncJobCallback(uuidAndTypeStateKey.getUuid(), asyncJobStatus));
asyncJobStatus.setState(AsyncJobState.COMPLETE);
getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
LOG.info("Completed job " + uuidAndTypeStateKey.getUuid());
LOG.debug("Completed job " + uuidAndTypeStateKey.getUuid());
return (result);
}
catch(Exception e)
@ -183,4 +183,15 @@ public class AsyncJobManager
// return TempFileStateProvider.getInstance();
}
/*******************************************************************************
**
*******************************************************************************/
public void cancelJob(String jobUUID)
{
Optional<AsyncJobStatus> jobStatus = getJobStatus(jobUUID);
jobStatus.ifPresent(asyncJobStatus -> asyncJobStatus.setCancelRequested(true));
}
}

24
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java
View File

@ -37,6 +37,8 @@ public class AsyncJobStatus implements Serializable
private Integer total;
private Exception caughtException;
private boolean cancelRequested;
/*******************************************************************************
@ -163,4 +165,26 @@ public class AsyncJobStatus implements Serializable
{
this.caughtException = caughtException;
}
/*******************************************************************************
** Getter for cancelRequested
**
*******************************************************************************/
public boolean getCancelRequested()
{
return cancelRequested;
}
/*******************************************************************************
** Setter for cancelRequested
**
*******************************************************************************/
public void setCancelRequested(boolean cancelRequested)
{
this.cancelRequested = cancelRequested;
}
}

195
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java Normal file
View File

@ -0,0 +1,195 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.async;
import java.io.Serializable;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Class that knows how to Run an asynchronous job (lambda, supplier) that writes into a
** RecordPipe, with another lambda (consumer) that consumes records from the pipe.
**
** Takes care of the job status monitoring, blocking when the pipe is empty, etc.
*******************************************************************************/
public class AsyncRecordPipeLoop
{
private static final Logger LOG = LogManager.getLogger(AsyncRecordPipeLoop.class);
private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000;
private static final int MAX_SLEEP_MS = 1000;
private static final int INIT_SLEEP_MS = 10;
private static final int MIN_RECORDS_TO_CONSUME = 10;
/*******************************************************************************
** Run an async-record-pipe-loop.
**
** @param jobName name for the async job thread
** @param recordLimit optionally, cancel the supplier/job after this number of records.
* e.g., for a preview step.
** @param recordPipe constructed before this call, and used in both of the lambdas
** @param supplier lambda that adds records into the pipe.
* e.g., a query or extract step.
** @param consumer lambda that consumes records from the pipe
* e.g., a transform/load step.
*******************************************************************************/
public int run(String jobName, Integer recordLimit, RecordPipe recordPipe, UnsafeFunction<AsyncJobCallback, ? extends Serializable> supplier, UnsafeSupplier<Integer> consumer) throws QException
{
///////////////////////////////////////////////////
// start the extraction function as an async job //
///////////////////////////////////////////////////
AsyncJobManager asyncJobManager = new AsyncJobManager();
String jobUUID = asyncJobManager.startJob(jobName, supplier::apply);
LOG.debug("Started supplier job [" + jobUUID + "] for record pipe.");
AsyncJobState jobState = AsyncJobState.RUNNING;
AsyncJobStatus asyncJobStatus = null;
int recordCount = 0;
int nextSleepMillis = INIT_SLEEP_MS;
long lastReceivedRecordsAt = System.currentTimeMillis();
long jobStartTime = System.currentTimeMillis();
while(jobState.equals(AsyncJobState.RUNNING))
{
if(recordPipe.countAvailableRecords() < MIN_RECORDS_TO_CONSUME)
{
///////////////////////////////////////////////////////////////
// if the pipe is too empty, sleep to let the producer work. //
// todo - smarter sleep? like get notified vs. sleep? //
///////////////////////////////////////////////////////////////
LOG.trace("Too few records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work");
SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS);
long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt;
if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS)
{
throw (new QException("Job appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago)."));
}
}
else
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. //
////////////////////////////////////////////////////////////////////////////////////////////////////////
lastReceivedRecordsAt = System.currentTimeMillis();
nextSleepMillis = INIT_SLEEP_MS;
recordCount += consumer.get();
LOG.debug(String.format("Processed %,d records so far", recordCount));
if(recordLimit != null && recordCount >= recordLimit)
{
asyncJobManager.cancelJob(jobUUID);
////////////////////////////////////////////////////////////////////////////////////////////////////
// in case the extract function doesn't recognize the cancellation request, //
// tell the pipe to "terminate" - meaning - flush its queue and just noop when given new records. //
// this should prevent anyone writing to such a pipe from potentially filling & blocking. //
////////////////////////////////////////////////////////////////////////////////////////////////////
recordPipe.terminate();
break;
}
}
//////////////////////////////
// refresh the job's status //
//////////////////////////////
Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(jobUUID);
if(optionalAsyncJobStatus.isEmpty())
{
/////////////////////////////////////////////////
// todo - ... maybe some version of try-again? //
/////////////////////////////////////////////////
throw (new QException("Could not get status of job [" + jobUUID + "]"));
}
asyncJobStatus = optionalAsyncJobStatus.get();
jobState = asyncJobStatus.getState();
}
LOG.debug("Job [" + jobUUID + "][" + jobName + "] completed with status: " + asyncJobStatus);
///////////////////////////////////
// propagate errors from the job //
///////////////////////////////////
if(asyncJobStatus != null && asyncJobStatus.getState().equals(AsyncJobState.ERROR))
{
throw (new QException("Job failed with an error", asyncJobStatus.getCaughtException()));
}
////////////////////////////////////////////
// send the final records to the consumer //
////////////////////////////////////////////
recordCount += consumer.get();
long endTime = System.currentTimeMillis();
if(recordCount > 0)
{
LOG.info(String.format("Processed %,d records", recordCount)
+ String.format(" at end of job [%s] in %,d ms (%.2f records/second).", jobName, (endTime - jobStartTime), 1000d * (recordCount / (.001d + (endTime - jobStartTime)))));
}
return (recordCount);
}
/*******************************************************************************
**
*******************************************************************************/
@FunctionalInterface
public interface UnsafeFunction<T, R>
{
/*******************************************************************************
**
*******************************************************************************/
R apply(T t) throws QException;
}
/*******************************************************************************
**
*******************************************************************************/
@FunctionalInterface
public interface UnsafeSupplier<T>
{
/*******************************************************************************
**
*******************************************************************************/
T get() throws QException;
}
}

99
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java Normal file
View File

@ -0,0 +1,99 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.automation;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
/*******************************************************************************
** enum of possible values for a record's Automation Status.
*******************************************************************************/
public enum AutomationStatus implements PossibleValueEnum<Integer>
{
PENDING_INSERT_AUTOMATIONS(1, "Pending Insert Automations"),
RUNNING_INSERT_AUTOMATIONS(2, "Running Insert Automations"),
FAILED_INSERT_AUTOMATIONS(3, "Failed Insert Automations"),
PENDING_UPDATE_AUTOMATIONS(4, "Pending Update Automations"),
RUNNING_UPDATE_AUTOMATIONS(5, "Running Update Automations"),
FAILED_UPDATE_AUTOMATIONS(6, "Failed Update Automations"),
OK(7, "OK");
private final Integer id;
private final String label;
/*******************************************************************************
**
*******************************************************************************/
AutomationStatus(int id, String label)
{
this.id = id;
this.label = label;
}
/*******************************************************************************
** Getter for id
**
*******************************************************************************/
public Integer getId()
{
return (id);
}
/*******************************************************************************
** Getter for label
**
*******************************************************************************/
public String getLabel()
{
return (label);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Integer getPossibleValueId()
{
return (getId());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getPossibleValueLabel()
{
return (getLabel());
}
}

40
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java Normal file
View File

@ -0,0 +1,40 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.automation;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
/*******************************************************************************
** Base class for custom-codes to run as an automation action
*******************************************************************************/
public abstract class RecordAutomationHandler
{
/*******************************************************************************
**
*******************************************************************************/
public abstract void execute(RecordAutomationInput recordAutomationInput) throws QException;
}

166
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java Normal file
View File

@ -0,0 +1,166 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.automation;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Utility class for updating the automation status data for records
*******************************************************************************/
public class RecordAutomationStatusUpdater
{
private static final Logger LOG = LogManager.getLogger(RecordAutomationStatusUpdater.class);
/*******************************************************************************
** for a list of records from a table, set their automation status - based on
** how the table is configured.
*******************************************************************************/
public static boolean setAutomationStatusInRecords(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus)
{
if(table == null || table.getAutomationDetails() == null || CollectionUtils.nullSafeIsEmpty(records))
{
return (false);
}
if(canWeSkipPendingAndGoToOkay(table, automationStatus))
{
automationStatus = AutomationStatus.OK;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// In case an automation is running, and it updates records - don't let those records be marked //
// as PENDING_UPDATE_AUTOMATIONS... this is meant to avoid having a record's automation update //
// itself, and then continue to do so in a loop (infinitely). //
// BUT - shouldn't this be allowed to update OTHER records to be pending updates? It seems like //
// yes - so -that'll probably be a bug to fix at some point in the future todo //
///////////////////////////////////////////////////////////////////////////////////////////////////
if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
{
Exception e = new Exception();
for(StackTraceElement stackTraceElement : e.getStackTrace())
{
String className = stackTraceElement.getClassName();
if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test"))
{
LOG.debug("Avoiding re-setting automation status to PENDING_UPDATE while running an automation");
return (false);
}
}
}
QTableAutomationDetails automationDetails = table.getAutomationDetails();
if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
{
for(QRecord record : records)
{
record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
// todo - another field - for the automation timestamp??
}
}
return (true);
}
/*******************************************************************************
** If a table has no automation actions defined for Insert (or Update), and we're
** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just
** move the status straight to OK.
*******************************************************************************/
private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
{
List<TableAutomationAction> tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>());
if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
{
return tableActions.stream().noneMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent()));
}
else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
{
return tableActions.stream().noneMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent()));
}
return (false);
}
/*******************************************************************************
** for a list of records, update their automation status and actually Update the
** backend as well.
*******************************************************************************/
public static void setAutomationStatusInRecordsAndUpdate(QInstance instance, QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus) throws QException
{
QTableAutomationDetails automationDetails = table.getAutomationDetails();
if(automationDetails != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
{
boolean didSetStatusField = setAutomationStatusInRecords(table, records, automationStatus);
if(didSetStatusField)
{
UpdateInput updateInput = new UpdateInput(instance);
updateInput.setSession(session);
updateInput.setTableName(table.getName());
/////////////////////////////////////////////////////////////////////////////////////
// build records with just their pkey & status field for this update, to avoid //
// changing other values (relies on assumption of Patch semantics in UpdateAction) //
/////////////////////////////////////////////////////////////////////////////////////
updateInput.setRecords(records.stream().map(r -> new QRecord()
.withTableName(r.getTableName())
.withValue(table.getPrimaryKeyField(), r.getValue(table.getPrimaryKeyField()))
.withValue(automationDetails.getStatusTracking().getFieldName(), r.getValue(automationDetails.getStatusTracking().getFieldName()))).toList());
updateInput.setAreAllValuesBeingUpdatedTheSame(true);
new UpdateAction().execute(updateInput);
}
}
else
{
// todo - verify if this is valid as other types are built
throw (new NotImplementedException("Updating record automation status is not implemented for table [" + table + "], tracking type: "
+ (automationDetails == null ? "null" : automationDetails.getStatusTracking().getType())));
}
}
}

227
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java Normal file
View File

@ -0,0 +1,227 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.automation.polling;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Singleton that runs a Polling Automation Provider. Call its 'start' method
** to make it go. Likely you need to set a sessionSupplier before you start -
** so that threads that do work will have a valid session.
*******************************************************************************/
public class PollingAutomationExecutor
{
private static final Logger LOG = LogManager.getLogger(PollingAutomationExecutor.class);
private static PollingAutomationExecutor pollingAutomationExecutor = null;
private Integer initialDelayMillis = 3000;
private Integer delayMillis = 1000;
private Supplier<QSession> sessionSupplier;
private RunningState runningState = RunningState.STOPPED;
private ScheduledExecutorService service;
/*******************************************************************************
** Singleton constructor
*******************************************************************************/
private PollingAutomationExecutor()
{
}
/*******************************************************************************
** Singleton accessor
*******************************************************************************/
public static PollingAutomationExecutor getInstance()
{
if(pollingAutomationExecutor == null)
{
pollingAutomationExecutor = new PollingAutomationExecutor();
}
return (pollingAutomationExecutor);
}
/*******************************************************************************
**
** @return true iff the schedule was started
*******************************************************************************/
public boolean start(QInstance instance, String providerName)
{
if(!runningState.equals(RunningState.STOPPED))
{
LOG.info("Request to start from an invalid running state [" + runningState + "]. Must be STOPPED.");
return (false);
}
LOG.info("Starting PollingAutomationExecutor");
service = Executors.newSingleThreadScheduledExecutor();
service.scheduleWithFixedDelay(new PollingAutomationRunner(instance, providerName, sessionSupplier), initialDelayMillis, delayMillis, TimeUnit.MILLISECONDS);
runningState = RunningState.RUNNING;
return (true);
}
/*******************************************************************************
** Stop, and don't wait to check if it worked or anything
*******************************************************************************/
public void stopAsync()
{
new Thread(this::stop).start();
}
/*******************************************************************************
** Issue a stop, and wait (a while) for it to succeed.
**
** @return true iff we see that the service fully stopped.
*******************************************************************************/
public boolean stop()
{
if(!runningState.equals(RunningState.RUNNING))
{
LOG.info("Request to stop from an invalid running state [" + runningState + "]. Must be RUNNING.");
return (false);
}
LOG.info("Stopping PollingAutomationExecutor");
runningState = RunningState.STOPPING;
service.shutdown();
try
{
if(service.awaitTermination(300, TimeUnit.SECONDS))
{
LOG.info("Successfully stopped PollingAutomationExecutor");
runningState = RunningState.STOPPED;
return (true);
}
LOG.info("Timed out waiting for service to fully terminate. Will be left in STOPPING state.");
}
catch(InterruptedException ie)
{
///////////////////////////////
// what does this ever mean? //
///////////////////////////////
}
return (false);
}
/*******************************************************************************
** Getter for initialDelayMillis
**
*******************************************************************************/
public Integer getInitialDelayMillis()
{
return initialDelayMillis;
}
/*******************************************************************************
** Setter for initialDelayMillis
**
*******************************************************************************/
public void setInitialDelayMillis(Integer initialDelayMillis)
{
this.initialDelayMillis = initialDelayMillis;
}
/*******************************************************************************
** Getter for delayMillis
**
*******************************************************************************/
public Integer getDelayMillis()
{
return delayMillis;
}
/*******************************************************************************
** Setter for delayMillis
**
*******************************************************************************/
public void setDelayMillis(Integer delayMillis)
{
this.delayMillis = delayMillis;
}
/*******************************************************************************
** Setter for sessionSupplier
**
*******************************************************************************/
public void setSessionSupplier(Supplier<QSession> sessionSupplier)
{
this.sessionSupplier = sessionSupplier;
}
/*******************************************************************************
** Getter for runningState
**
*******************************************************************************/
public RunningState getRunningState()
{
return runningState;
}
/*******************************************************************************
**
*******************************************************************************/
public enum RunningState
{
STOPPED,
RUNNING,
STOPPING,
}
}

390
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java Normal file
View File

@ -0,0 +1,390 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.automation.polling;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Runnable for the Polling Automation Provider, that looks for records that
** need automations, and executes them.
*******************************************************************************/
class PollingAutomationRunner implements Runnable
{
private static final Logger LOG = LogManager.getLogger(PollingAutomationRunner.class);
private QInstance instance;
private String providerName;
private Supplier<QSession> sessionSupplier;
private List<QTableMetaData> managedTables = new ArrayList<>();
private Map<String, List<TableAutomationAction>> tableInsertActions = new HashMap<>();
private Map<String, List<TableAutomationAction>> tableUpdateActions = new HashMap<>();
private Map<String, Map<AutomationStatus, List<TableAutomationAction>>> tableActions = new HashMap<>();
private static Map<TriggerEvent, AutomationStatus> triggerEventAutomationStatusMap = Map.of(
TriggerEvent.POST_INSERT, AutomationStatus.PENDING_INSERT_AUTOMATIONS,
TriggerEvent.POST_UPDATE, AutomationStatus.PENDING_UPDATE_AUTOMATIONS
);
private static Map<AutomationStatus, AutomationStatus> pendingToRunningStatusMap = Map.of(
AutomationStatus.PENDING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS,
AutomationStatus.PENDING_UPDATE_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS
);
private static Map<AutomationStatus, AutomationStatus> pendingToFailedStatusMap = Map.of(
AutomationStatus.PENDING_INSERT_AUTOMATIONS, AutomationStatus.FAILED_INSERT_AUTOMATIONS,
AutomationStatus.PENDING_UPDATE_AUTOMATIONS, AutomationStatus.FAILED_UPDATE_AUTOMATIONS
);
/*******************************************************************************
**
*******************************************************************************/
public PollingAutomationRunner(QInstance instance, String providerName, Supplier<QSession> sessionSupplier)
{
this.instance = instance;
this.providerName = providerName;
this.sessionSupplier = sessionSupplier;
//////////////////////////////////////////////////////////////////////
// todo - share logic like this among any automation implementation //
//////////////////////////////////////////////////////////////////////
for(QTableMetaData table : instance.getTables().values())
{
if(table.getAutomationDetails() != null && this.providerName.equals(table.getAutomationDetails().getProviderName()))
{
managedTables.add(table);
///////////////////////////////////////////////////////////////////////////
// organize the table's actions by type //
// todo - in future, need user-defined actions here too (and refreshed!) //
///////////////////////////////////////////////////////////////////////////
for(TableAutomationAction action : table.getAutomationDetails().getActions())
{
AutomationStatus automationStatus = triggerEventAutomationStatusMap.get(action.getTriggerEvent());
tableActions.putIfAbsent(table.getName(), new HashMap<>());
tableActions.get(table.getName()).putIfAbsent(automationStatus, new ArrayList<>());
tableActions.get(table.getName()).get(automationStatus).add(action);
}
//////////////////////////////
// sort actions by priority //
//////////////////////////////
if(tableInsertActions.containsKey(table.getName()))
{
tableInsertActions.get(table.getName()).sort(Comparator.comparing(TableAutomationAction::getPriority));
}
if(tableUpdateActions.containsKey(table.getName()))
{
tableUpdateActions.get(table.getName()).sort(Comparator.comparing(TableAutomationAction::getPriority));
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run()
{
Thread.currentThread().setName(getClass().getSimpleName() + ">" + providerName);
LOG.info("Running " + this.getClass().getSimpleName() + "[providerName=" + providerName + "]");
for(QTableMetaData table : managedTables)
{
try
{
processTable(table);
}
catch(Exception e)
{
LOG.error("Error processing automations on table: " + table, e);
}
}
}
/*******************************************************************************
** Query for and process records that have a PENDING status on a given table.
*******************************************************************************/
private void processTable(QTableMetaData table) throws QException
{
QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession();
processTableInsertOrUpdate(table, session, AutomationStatus.PENDING_INSERT_AUTOMATIONS);
processTableInsertOrUpdate(table, session, AutomationStatus.PENDING_UPDATE_AUTOMATIONS);
}
/*******************************************************************************
** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table.
*******************************************************************************/
private void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException
{
List<TableAutomationAction> actions = tableActions
.getOrDefault(table.getName(), Collections.emptyMap())
.getOrDefault(automationStatus, Collections.emptyList());
if(CollectionUtils.nullSafeIsEmpty(actions))
{
return;
}
LOG.debug(" Query for records " + automationStatus + " in " + table);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// run an async-pipe loop - that will query for records in PENDING - put them in a pipe - then apply actions to them //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
RecordPipe recordPipe = new RecordPipe();
AsyncRecordPipeLoop asyncRecordPipeLoop = new AsyncRecordPipeLoop();
asyncRecordPipeLoop.run("PollingAutomationRunner>Query>" + automationStatus, null, recordPipe, (status) ->
{
QueryInput queryInput = new QueryInput(instance);
queryInput.setSession(session);
queryInput.setTableName(table.getName());
AutomationStatusTrackingType statusTrackingType = table.getAutomationDetails().getStatusTracking().getType();
if(AutomationStatusTrackingType.FIELD_IN_TABLE.equals(statusTrackingType))
{
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId()))));
}
else
{
throw (new NotImplementedException("Automation Status Tracking type [" + statusTrackingType + "] is not yet implemented in here."));
}
queryInput.setRecordPipe(recordPipe);
return (new QueryAction().execute(queryInput));
}, () ->
{
List<QRecord> records = recordPipe.consumeAvailableRecords();
applyActionsToRecords(session, table, records, actions, automationStatus);
return (records.size());
}
);
}
/*******************************************************************************
** For a set of records that were found to be in a PENDING state - run all the
** table's actions against them - IF they are found to match the action's filter
** (assuming it has one - if it doesn't, then all records match).
*******************************************************************************/
private void applyActionsToRecords(QSession session, QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
{
if(CollectionUtils.nullSafeIsEmpty(records))
{
return;
}
///////////////////////////////////////////////////
// mark the records as RUNNING their automations //
///////////////////////////////////////////////////
RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToRunningStatusMap.get(automationStatus));
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
// foreach action - run it against the records (but only if they match the action's filter, if there is one) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
boolean anyActionsFailed = false;
for(TableAutomationAction action : actions)
{
try
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(session, table, records, action);
LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
if(CollectionUtils.nullSafeHasContents(matchingQRecords))
{
LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
applyActionToMatchingRecords(session, table, matchingQRecords, action);
}
}
catch(Exception e)
{
LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
anyActionsFailed = true;
}
}
////////////////////////////////////////
// update status on all these records //
////////////////////////////////////////
if(anyActionsFailed)
{
RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus));
}
else
{
RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK);
}
}
/*******************************************************************************
** For a given action, and a list of records - return a new list, of the ones
** which match the action's filter (if there is one - if not, then all match).
**
** Note that this WILL re-query the objects (ALWAYS - even if the action has no filter).
** This has the nice side effect of always giving fresh/updated records, despite having
** some cost.
**
** At one point, we considered just applying the filter using java-comparisons,
** but that will almost certainly give potentially different results than a true
** backend - e.g., just consider if the DB is case-sensitive for strings...
*******************************************************************************/
private List<QRecord> getRecordsMatchingActionFilter(QSession session, QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws QException
{
QueryInput queryInput = new QueryInput(instance);
queryInput.setSession(session);
queryInput.setTableName(table.getName());
QQueryFilter filter = new QQueryFilter();
/////////////////////////////////////////////////////////////////////////////////////////////////////
// copy filter criteria from the action's filter to a new filter that we'll run here. //
// Critically - don't modify the filter object on the action! as that object has a long lifespan. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
if(action.getFilter() != null)
{
if(action.getFilter().getCriteria() != null)
{
action.getFilter().getCriteria().forEach(filter::addCriteria);
}
if(action.getFilter().getOrderBys() != null)
{
action.getFilter().getOrderBys().forEach(filter::addOrderBy);
}
}
filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList()));
/////////////////////////////////////////////////////////////////////////////////////////////
// always add order-by the primary key, to give more predictable/consistent results //
// todo - in future - if this becomes a source of slowness, make this a config to opt-out? //
/////////////////////////////////////////////////////////////////////////////////////////////
filter.addOrderBy(new QFilterOrderBy().withFieldName(table.getPrimaryKeyField()));
queryInput.setFilter(filter);
return (new QueryAction().execute(queryInput).getRecords());
}
/*******************************************************************************
** Finally, actually run action code against a list of known matching records.
*******************************************************************************/
private void applyActionToMatchingRecords(QSession session, QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
{
if(StringUtils.hasContent(action.getProcessName()))
{
/////////////////////////////////////////////////////////////////////////////////////////
// if the action has a process associated with it - run that process. //
// tell it to SKIP frontend steps. //
// give the process a callback w/ a query filter that has the p-keys of these records. //
/////////////////////////////////////////////////////////////////////////////////////////
RunProcessInput runProcessInput = new RunProcessInput(instance);
runProcessInput.setSession(session);
runProcessInput.setProcessName(action.getProcessName());
runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
runProcessInput.setCallback(new QProcessCallback()
{
@Override
public QQueryFilter getQueryFilter()
{
List<Serializable> recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList());
return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds)));
}
});
RunProcessAction runProcessAction = new RunProcessAction();
RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput);
if(runProcessOutput.getException().isPresent())
{
throw (runProcessOutput.getException().get());
}
}
else if(action.getCodeReference() != null)
{
LOG.debug(" Executing action: [" + action.getName() + "] as code reference: " + action.getCodeReference());
RecordAutomationInput input = new RecordAutomationInput(instance);
input.setSession(session);
input.setTableName(table.getName());
input.setRecordList(records);
RecordAutomationHandler recordAutomationHandler = QCodeLoader.getRecordAutomationHandler(action);
recordAutomationHandler.execute(input);
}
}
}

118
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java
View File

@ -24,12 +24,15 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
import java.util.Optional;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -97,6 +100,120 @@ public class QCodeLoader
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public static <T extends BackendStep> T getBackendStep(Class<T> expectedType, QCodeReference codeReference)
{
if(codeReference == null)
{
return (null);
}
if(!codeReference.getCodeType().equals(QCodeType.JAVA))
{
///////////////////////////////////////////////////////////////////////////////////////
// todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
///////////////////////////////////////////////////////////////////////////////////////
throw (new IllegalArgumentException("Only JAVA BackendSteps are supported at this time."));
}
try
{
Class<?> customizerClass = Class.forName(codeReference.getName());
return ((T) customizerClass.getConstructor().newInstance());
}
catch(Exception e)
{
LOG.error("Error initializing customizer: " + codeReference);
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// return null here - under the assumption that during normal run-time operations, we'll never hit here //
// as we'll want to validate all functions in the instance validator at startup time (and IT will throw //
// if it finds an invalid code reference //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
return (null);
}
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public static <T> T getAdHoc(Class<T> expectedType, QCodeReference codeReference)
{
if(codeReference == null)
{
return (null);
}
if(!codeReference.getCodeType().equals(QCodeType.JAVA))
{
///////////////////////////////////////////////////////////////////////////////////////
// todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
///////////////////////////////////////////////////////////////////////////////////////
throw (new IllegalArgumentException("Only JAVA code references are supported at this time."));
}
try
{
Class<?> customizerClass = Class.forName(codeReference.getName());
return ((T) customizerClass.getConstructor().newInstance());
}
catch(Exception e)
{
LOG.error("Error initializing customizer: " + codeReference);
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// return null here - under the assumption that during normal run-time operations, we'll never hit here //
// as we'll want to validate all functions in the instance validator at startup time (and IT will throw //
// if it finds an invalid code reference //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
return (null);
}
}
/*******************************************************************************
**
*******************************************************************************/
public static RecordAutomationHandler getRecordAutomationHandler(TableAutomationAction action) throws QException
{
try
{
QCodeReference codeReference = action.getCodeReference();
if(!codeReference.getCodeType().equals(QCodeType.JAVA))
{
///////////////////////////////////////////////////////////////////////////////////////
// todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
///////////////////////////////////////////////////////////////////////////////////////
throw (new IllegalArgumentException("Only JAVA customizers are supported at this time."));
}
Class<?> codeClass = Class.forName(codeReference.getName());
Object codeObject = codeClass.getConstructor().newInstance();
if(!(codeObject instanceof RecordAutomationHandler recordAutomationHandler))
{
throw (new QException("The supplied code [" + codeClass.getName() + "] is not an instance of RecordAutomationHandler"));
}
return (recordAutomationHandler);
}
catch(QException qe)
{
throw (qe);
}
catch(Exception e)
{
throw (new QException("Error getting record automation handler for action [" + action.getName() + "]", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -121,4 +238,5 @@ public class QCodeLoader
throw (new QException("Error getting custom possible value provider for PVS [" + possibleValueSource.getName() + "]", e));
}
}
}

21
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java
View File

@ -1,3 +1,24 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.customizers;

22
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java
View File

@ -1,3 +1,24 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.customizers;
@ -24,6 +45,7 @@ public enum TableCustomizers
{
POST_QUERY_RECORD(new TableCustomizer("postQueryRecord", Function.class, ((Object x) ->
{
@SuppressWarnings("unchecked")
Function<QRecord, QRecord> function = (Function<QRecord, QRecord>) x;
QRecord output = function.apply(new QRecord());
})));

43
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java Normal file
View File

@ -0,0 +1,43 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.dashboard;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
import com.kingsrook.qqq.backend.core.model.session.QSession;
/*******************************************************************************
** Base class for rendering qqq dashboard widgets
**
*******************************************************************************/
public abstract class AbstractWidgetRenderer
{
/*******************************************************************************
**
*******************************************************************************/
public abstract Object render(QInstance qInstance, QSession session, QWidgetMetaDataInterface qWidgetMetaData) throws QException;
}

100
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java Normal file
View File

@ -0,0 +1,100 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.dashboard;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QuickSightChart;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QuickSightChartMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.quicksight.QuickSightClient;
import software.amazon.awssdk.services.quicksight.model.GenerateEmbedUrlForRegisteredUserRequest;
import software.amazon.awssdk.services.quicksight.model.GenerateEmbedUrlForRegisteredUserResponse;
import software.amazon.awssdk.services.quicksight.model.RegisteredUserDashboardEmbeddingConfiguration;
import software.amazon.awssdk.services.quicksight.model.RegisteredUserEmbeddingExperienceConfiguration;
/*******************************************************************************
** Widget implementation for amazon QuickSight charts
**
*******************************************************************************/
public class QuickSightChartRenderer extends AbstractWidgetRenderer
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public Object render(QInstance qInstance, QSession session, QWidgetMetaDataInterface metaData) throws QException
{
try
{
QuickSightChartMetaData quickSightMetaData = (QuickSightChartMetaData) metaData;
QuickSightClient quickSightClient = getQuickSightClient(quickSightMetaData);
final RegisteredUserEmbeddingExperienceConfiguration experienceConfiguration = RegisteredUserEmbeddingExperienceConfiguration.builder()
.dashboard(
RegisteredUserDashboardEmbeddingConfiguration.builder()
.initialDashboardId(quickSightMetaData.getDashboardId())
.build())
.build();
final GenerateEmbedUrlForRegisteredUserRequest generateEmbedUrlForRegisteredUserRequest = GenerateEmbedUrlForRegisteredUserRequest.builder()
.awsAccountId(quickSightMetaData.getAccountId())
.userArn(quickSightMetaData.getUserArn())
.experienceConfiguration(experienceConfiguration)
.build();
final GenerateEmbedUrlForRegisteredUserResponse generateEmbedUrlForRegisteredUserResponse = quickSightClient.generateEmbedUrlForRegisteredUser(generateEmbedUrlForRegisteredUserRequest);
String embedUrl = generateEmbedUrlForRegisteredUserResponse.embedUrl();
return (new QuickSightChart(metaData.getName(), quickSightMetaData.getLabel(), embedUrl));
}
catch(Exception e)
{
throw (new QException("Error rendering widget", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
private QuickSightClient getQuickSightClient(QuickSightChartMetaData metaData)
{
AwsBasicCredentials awsCredentials = AwsBasicCredentials.create(metaData.getAccessKey(), metaData.getSecretKey());
QuickSightClient amazonQuickSightClient = QuickSightClient.builder()
.credentialsProvider(StaticCredentialsProvider.create(awsCredentials))
.region(Region.of(metaData.getRegion()))
.build();
return (amazonQuickSightClient);
}
}

49
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java Normal file
View File

@ -0,0 +1,49 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.dashboard;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
import com.kingsrook.qqq.backend.core.model.session.QSession;
/*******************************************************************************
** Class for loading widget implementation code and rendering of widgets
**
*******************************************************************************/
public class WidgetDataLoader
{
/*******************************************************************************
**
*******************************************************************************/
public Object execute(QInstance qInstance, QSession session, String name) throws QException
{
QWidgetMetaDataInterface widget = qInstance.getWidget(name);
AbstractWidgetRenderer widgetRenderer = QCodeLoader.getAdHoc(AbstractWidgetRenderer.class, widget.getCodeReference());
Object w = widgetRenderer.render(qInstance, session, widget);
return (widgetRenderer.render(qInstance, session, widget));
}
}

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java
View File

@ -22,7 +22,6 @@
package com.kingsrook.qqq.backend.core.actions.interfaces;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@ -32,19 +31,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
** Interface for the Insert action.
**
*******************************************************************************/
public interface InsertInterface
public interface InsertInterface extends QActionInterface
{
/*******************************************************************************
**
*******************************************************************************/
InsertOutput execute(InsertInput insertInput) throws QException;
/*******************************************************************************
**
*******************************************************************************/
default QBackendTransaction openTransaction(InsertInput insertInput) throws QException
{
return (new QBackendTransaction());
}
}

23
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java
View File

@ -19,31 +19,26 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
package com.kingsrook.qqq.backend.core.actions.interfaces;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
/*******************************************************************************
** Backend step to receive a bulk-insert upload file
**
*******************************************************************************/
public class BulkInsertReceiveFileStep implements BackendStep
public interface QActionInterface
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
default QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException
{
List<QRecord> qRecords = BulkInsertUtils.getQRecordsFromFile(runBackendStepInput);
runBackendStepOutput.addValue("noOfFileRows", qRecords.size());
runBackendStepOutput.setRecords(qRecords);
return (new QBackendTransaction());
}
}

8
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java
View File

@ -109,6 +109,14 @@ public class MetaDataAction
}
metaDataOutput.setAppTree(appTree);
////////////////////////////////////
// add branding metadata if found //
////////////////////////////////////
if(metaDataInput.getInstance().getBranding() != null)
{
metaDataOutput.setBranding(metaDataInput.getInstance().getBranding());
}
// todo post-customization - can do whatever w/ the result if you want?
return metaDataOutput;

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallback.java
View File

@ -39,10 +39,16 @@ public interface QProcessCallback
/*******************************************************************************
** Get the filter query for this callback.
*******************************************************************************/
QQueryFilter getQueryFilter();
default QQueryFilter getQueryFilter()
{
return (null);
}
/*******************************************************************************
** Get the field values for this callback.
*******************************************************************************/
Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields);
default Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
{
return (null);
}
}

89
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java
View File

@ -82,15 +82,28 @@ public class RunProcessAction
runProcessOutput.setProcessUUID(runProcessInput.getProcessUUID());
UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS);
ProcessState processState = primeProcessState(runProcessInput, stateKey);
ProcessState processState = primeProcessState(runProcessInput, stateKey, process);
// todo - custom routing
List<QStepMetaData> stepList = getAvailableStepList(process, runProcessInput);
try
{
String lastStepName = runProcessInput.getStartAfterStep();
STEP_LOOP:
for(QStepMetaData step : stepList)
while(true)
{
///////////////////////////////////////////////////////////////////////////////////////////////////////
// always refresh the step list - as any step that runs can modify it (in the process state). //
// this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
List<QStepMetaData> stepList = getAvailableStepList(processState, process, lastStepName);
if(stepList.isEmpty())
{
break;
}
QStepMetaData step = stepList.get(0);
lastStepName = step.getName();
if(step instanceof QFrontendStepMetaData)
{
////////////////////////////////////////////////////////////////
@ -127,6 +140,7 @@ public class RunProcessAction
///////////////////////
// Run backend steps //
///////////////////////
LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
}
else
@ -169,7 +183,7 @@ public class RunProcessAction
** When we start running a process (or resuming it), get data in the RunProcessRequest
** either from the state provider (if they're found, for a resume).
*******************************************************************************/
ProcessState primeProcessState(RunProcessInput runProcessInput, UUIDAndTypeStateKey stateKey) throws QException
ProcessState primeProcessState(RunProcessInput runProcessInput, UUIDAndTypeStateKey stateKey, QProcessMetaData process) throws QException
{
Optional<ProcessState> optionalProcessState = loadState(stateKey);
if(optionalProcessState.isEmpty())
@ -177,11 +191,14 @@ public class RunProcessAction
if(runProcessInput.getStartAfterStep() == null)
{
///////////////////////////////////////////////////////////////////////////////////
// this is fine - it means its our first time running in the backend. //
// This condition (no state in state-provider, and no start-after-step) means //
// that we're starting a new process! Init the process state here, then //
// Go ahead and store the state that we have (e.g., w/ initial records & values) //
///////////////////////////////////////////////////////////////////////////////////
storeState(stateKey, runProcessInput.getProcessState());
optionalProcessState = Optional.of(runProcessInput.getProcessState());
ProcessState processState = runProcessInput.getProcessState();
processState.setStepList(process.getStepList().stream().map(QStepMetaData::getName).toList());
storeState(stateKey, processState);
optionalProcessState = Optional.of(processState);
}
else
{
@ -249,41 +266,63 @@ public class RunProcessAction
/*******************************************************************************
** Get the list of steps which are eligible to run.
*******************************************************************************/
private List<QStepMetaData> getAvailableStepList(QProcessMetaData process, RunProcessInput runProcessInput)
private List<QStepMetaData> getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep) throws QException
{
if(runProcessInput.getStartAfterStep() == null)
if(lastStep == null)
{
/////////////////////////////////////////////////////////////////////////////
// if the caller did not supply a 'startAfterStep', then use the full list //
/////////////////////////////////////////////////////////////////////////////
return (process.getStepList());
///////////////////////////////////////////////////////////////////////
// if the caller did not supply a 'lastStep', then use the full list //
///////////////////////////////////////////////////////////////////////
return (stepNamesToSteps(process, processState.getStepList()));
}
else
{
////////////////////////////////////////////////////////////////////////////////
// else, loop until the startAfterStep is found, and return the ones after it //
////////////////////////////////////////////////////////////////////////////////
boolean foundStartAfterStep = false;
List<QStepMetaData> rs = new ArrayList<>();
////////////////////////////////////////////////////////////////////////////
// else, loop until the 'lastStep' is found, and return the ones after it //
////////////////////////////////////////////////////////////////////////////
boolean foundLastStep = false;
List<String> validStepNames = new ArrayList<>();
for(QStepMetaData step : process.getStepList())
for(String stepName : processState.getStepList())
{
if(foundStartAfterStep)
if(foundLastStep)
{
rs.add(step);
validStepNames.add(stepName);
}
if(step.getName().equals(runProcessInput.getStartAfterStep()))
if(stepName.equals(lastStep))
{
foundStartAfterStep = true;
foundLastStep = true;
}
}
return (rs);
return (stepNamesToSteps(process, validStepNames));
}
}
/*******************************************************************************
**
*******************************************************************************/
private List<QStepMetaData> stepNamesToSteps(QProcessMetaData process, List<String> stepNames) throws QException
{
List<QStepMetaData> result = new ArrayList<>();
for(String stepName : stepNames)
{
QStepMetaData step = process.getStep(stepName);
if(step == null)
{
throw(new QException("Could not find a step named [" + stepName + "] in this process."));
}
result.add(step);
}
return (result);
}
/*******************************************************************************
** Load an instance of the appropriate state provider
**

55
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java
View File

@ -41,8 +41,13 @@ public class RecordPipe
{
private static final Logger LOG = LogManager.getLogger(RecordPipe.class);
private static final long BLOCKING_SLEEP_MILLIS = 100;
private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes
private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(1_000);
private boolean isTerminated = false;
private Consumer<List<QRecord>> postRecordActions = null;
/////////////////////////////////////
@ -51,11 +56,31 @@ public class RecordPipe
private List<QRecord> singleRecordListForPostRecordActions = new ArrayList<>();
/*******************************************************************************
** Add a record to the pipe. Will block if the pipe is full.
** Turn off the pipe. Stop accepting new records (just ignore them in the add
** method). Clear the existing queue. Don't return any more records. Note that
** if consumeAvailableRecords was running in another thread, it may still return
** some records that it read before this call.
*******************************************************************************/
public void terminate()
{
isTerminated = true;
queue.clear();
}
/*******************************************************************************
** Add a record to the pipe. Will block if the pipe is full. Will noop if pipe is terminated.
*******************************************************************************/
public void addRecord(QRecord record)
{
if(isTerminated)
{
return;
}
if(postRecordActions != null)
{
////////////////////////////////////////////////////////////////////////////////////
@ -82,18 +107,29 @@ public class RecordPipe
{
boolean offerResult = queue.offer(record);
while(!offerResult)
if(!offerResult && !isTerminated)
{
LOG.debug("Record pipe.add failed (due to full pipe). Blocking.");
SleepUtils.sleep(100, TimeUnit.MILLISECONDS);
offerResult = queue.offer(record);
long sleepLoopStartTime = System.currentTimeMillis();
long now = System.currentTimeMillis();
while(!offerResult && !isTerminated)
{
if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS)
{
LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS);
throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long."));
}
LOG.trace("Record pipe.add failed (due to full pipe). Blocking.");
SleepUtils.sleep(BLOCKING_SLEEP_MILLIS, TimeUnit.MILLISECONDS);
offerResult = queue.offer(record);
now = System.currentTimeMillis();
}
}
}
/*******************************************************************************
** Add a list of records to the pipe
** Add a list of records to the pipe. Will block if the pipe is full. Will noop if pipe is terminated.
*******************************************************************************/
public void addRecords(List<QRecord> records)
{
@ -117,7 +153,7 @@ public class RecordPipe
{
List<QRecord> rs = new ArrayList<>();
while(true)
while(!isTerminated)
{
QRecord record = queue.poll();
if(record == null)
@ -137,6 +173,11 @@ public class RecordPipe
*******************************************************************************/
public int countAvailableRecords()
{
if(isTerminated)
{
return (0);
}
return (queue.size());
}

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@ -48,6 +50,9 @@ public class InsertAction
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);
setAutomationStatusField(insertInput);
QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
// todo pre-customization - just get to modify the request?
InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput);
@ -57,6 +62,16 @@ public class InsertAction
/*******************************************************************************
** If the table being inserted into uses an automation-status field, populate it now.
*******************************************************************************/
private void setAutomationStatusField(InsertInput insertInput)
{
RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS);
}
/*******************************************************************************
**
*******************************************************************************/

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java
View File

@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
@ -42,6 +44,7 @@ public class UpdateAction
public UpdateOutput execute(UpdateInput updateInput) throws QException
{
ActionHelper.validateSession(updateInput);
setAutomationStatusField(updateInput);
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(updateInput.getBackend());
@ -50,4 +53,15 @@ public class UpdateAction
// todo post-customization - can do whatever w/ the result if you want
return updateResult;
}
/*******************************************************************************
** If the table being updated uses an automation-status field, populate it now.
*******************************************************************************/
private void setAutomationStatusField(UpdateInput updateInput)
{
RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS);
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java
View File

@ -89,7 +89,7 @@ public class QPossibleValueTranslator
*******************************************************************************/
public void translatePossibleValuesInRecords(QTableMetaData table, List<QRecord> records)
{
if(records == null)
if(records == null || table == null)
{
return;
}

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
View File

@ -74,6 +74,10 @@ public class QValueFormatter
{
return formatValue(field, ValueUtils.getValueAsBigDecimal(value));
}
else if(e.getMessage().equals("f != java.lang.String"))
{
return formatValue(field, ValueUtils.getValueAsBigDecimal(value));
}
else if(e.getMessage().equals("d != java.math.BigDecimal"))
{
return formatValue(field, ValueUtils.getValueAsInteger(value));

288
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java
View File

@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
@ -62,8 +63,7 @@ public class CsvToQRecordAdapter
*******************************************************************************/
public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
{
this.recordPipe = recordPipe;
doBuildRecordsFromCsv(csv, table, mapping, recordCustomizer);
buildRecordsFromCsv(new InputWrapper().withRecordPipe(recordPipe).withCsv(csv).withTable(table).withMapping(mapping).withRecordCustomizer(recordCustomizer));
}
@ -75,8 +75,7 @@ public class CsvToQRecordAdapter
*******************************************************************************/
public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
{
this.recordList = new ArrayList<>();
doBuildRecordsFromCsv(csv, table, mapping, null);
buildRecordsFromCsv(new InputWrapper().withCsv(csv).withTable(table).withMapping(mapping));
return (recordList);
}
@ -88,13 +87,29 @@ public class CsvToQRecordAdapter
**
** todo - meta-data validation, type handling
*******************************************************************************/
public void doBuildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
public void buildRecordsFromCsv(InputWrapper inputWrapper)
{
String csv = inputWrapper.getCsv();
AbstractQFieldMapping<?> mapping = inputWrapper.getMapping();
Consumer<QRecord> recordCustomizer = inputWrapper.getRecordCustomizer();
QTableMetaData table = inputWrapper.getTable();
Integer limit = inputWrapper.getLimit();
if(!StringUtils.hasContent(csv))
{
throw (new IllegalArgumentException("Empty csv value was provided."));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// if caller supplied a record pipe, use it -- but if it's null, then create a recordList to populate. //
// see addRecord method for usage. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
this.recordPipe = inputWrapper.getRecordPipe();
if(this.recordPipe == null)
{
this.recordList = new ArrayList<>();
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// once, from a DOS csv file (that had come from Excel), we had a "" character (FEFF, Byte-order marker) at the start of a //
// CSV, which caused our first header to not match... So, let us strip away any FEFF or FFFE's at the start of CSV strings. //
@ -120,9 +135,12 @@ public class CsvToQRecordAdapter
List<String> headers = csvParser.getHeaderNames();
headers = makeHeadersUnique(headers);
List<CSVRecord> csvRecords = csvParser.getRecords();
for(CSVRecord csvRecord : csvRecords)
Iterator<CSVRecord> csvIterator = csvParser.iterator();
int recordCount = 0;
while(csvIterator.hasNext())
{
CSVRecord csvRecord = csvIterator.next();
//////////////////////////////////////////////////////////////////
// put values from the CSV record into a map of header -> value //
//////////////////////////////////////////////////////////////////
@ -144,6 +162,12 @@ public class CsvToQRecordAdapter
runRecordCustomizer(recordCustomizer, qRecord);
addRecord(qRecord);
recordCount++;
if(limit != null && recordCount > limit)
{
break;
}
}
}
else if(AbstractQFieldMapping.SourceType.INDEX.equals(mapping.getSourceType()))
@ -155,9 +179,12 @@ public class CsvToQRecordAdapter
CSVFormat.DEFAULT
.withTrim());
List<CSVRecord> csvRecords = csvParser.getRecords();
for(CSVRecord csvRecord : csvRecords)
Iterator<CSVRecord> csvIterator = csvParser.iterator();
int recordCount = 0;
while(csvIterator.hasNext())
{
CSVRecord csvRecord = csvIterator.next();
/////////////////////////////////////////////////////////////////
// put values from the CSV record into a map of index -> value //
/////////////////////////////////////////////////////////////////
@ -180,6 +207,12 @@ public class CsvToQRecordAdapter
runRecordCustomizer(recordCustomizer, qRecord);
addRecord(qRecord);
recordCount++;
if(limit != null && recordCount > limit)
{
break;
}
}
}
else
@ -261,4 +294,241 @@ public class CsvToQRecordAdapter
}
}
/*******************************************************************************
** Getter for recordList - note - only is valid if you don't supply a pipe in
** the input. If you do supply a pipe, then you get an exception if you call here!
**
*******************************************************************************/
public List<QRecord> getRecordList()
{
if(recordPipe != null)
{
throw (new IllegalStateException("getRecordList called on a CSVToQRecordAdapter that ran with a recordPipe."));
}
return recordList;
}
/*******************************************************************************
**
*******************************************************************************/
public static class InputWrapper
{
private RecordPipe recordPipe;
private String csv;
private QTableMetaData table;
private AbstractQFieldMapping<?> mapping;
private Consumer<QRecord> recordCustomizer;
private Integer limit;
/*******************************************************************************
** Getter for recordPipe
**
*******************************************************************************/
public RecordPipe getRecordPipe()
{
return recordPipe;
}
/*******************************************************************************
** Setter for recordPipe
**
*******************************************************************************/
public void setRecordPipe(RecordPipe recordPipe)
{
this.recordPipe = recordPipe;
}
/*******************************************************************************
** Fluent setter for recordPipe
**
*******************************************************************************/
public InputWrapper withRecordPipe(RecordPipe recordPipe)
{
this.recordPipe = recordPipe;
return (this);
}
/*******************************************************************************
** Getter for csv
**
*******************************************************************************/
public String getCsv()
{
return csv;
}
/*******************************************************************************
** Setter for csv
**
*******************************************************************************/
public void setCsv(String csv)
{
this.csv = csv;
}
/*******************************************************************************
** Fluent setter for csv
**
*******************************************************************************/
public InputWrapper withCsv(String csv)
{
this.csv = csv;
return (this);
}
/*******************************************************************************
** Getter for table
**
*******************************************************************************/
public QTableMetaData getTable()
{
return table;
}
/*******************************************************************************
** Setter for table
**
*******************************************************************************/
public void setTable(QTableMetaData table)
{
this.table = table;
}
/*******************************************************************************
** Fluent setter for table
**
*******************************************************************************/
public InputWrapper withTable(QTableMetaData table)
{
this.table = table;
return (this);
}
/*******************************************************************************
** Getter for mapping
**
*******************************************************************************/
public AbstractQFieldMapping<?> getMapping()
{
return mapping;
}
/*******************************************************************************
** Setter for mapping
**
*******************************************************************************/
public void setMapping(AbstractQFieldMapping<?> mapping)
{
this.mapping = mapping;
}
/*******************************************************************************
** Fluent setter for mapping
**
*******************************************************************************/
public InputWrapper withMapping(AbstractQFieldMapping<?> mapping)
{
this.mapping = mapping;
return (this);
}
/*******************************************************************************
** Getter for recordCustomizer
**
*******************************************************************************/
public Consumer<QRecord> getRecordCustomizer()
{
return recordCustomizer;
}
/*******************************************************************************
** Setter for recordCustomizer
**
*******************************************************************************/
public void setRecordCustomizer(Consumer<QRecord> recordCustomizer)
{
this.recordCustomizer = recordCustomizer;
}
/*******************************************************************************
** Fluent setter for recordCustomizer
**
*******************************************************************************/
public InputWrapper withRecordCustomizer(Consumer<QRecord> recordCustomizer)
{
this.recordCustomizer = recordCustomizer;
return (this);
}
/*******************************************************************************
** Getter for limit
**
*******************************************************************************/
public Integer getLimit()
{
return limit;
}
/*******************************************************************************
** Setter for limit
**
*******************************************************************************/
public void setLimit(Integer limit)
{
this.limit = limit;
}
/*******************************************************************************
** Fluent setter for limit
**
*******************************************************************************/
public InputWrapper withLimit(Integer limit)
{
this.limit = limit;
return (this);
}
}
}

198
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -22,38 +22,39 @@
package com.kingsrook.qqq.backend.core.instances;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteStoreStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditReceiveValuesStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditStoreRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertStoreRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.logging.log4j.LogManager;
@ -293,6 +294,20 @@ public class QInstanceEnricher
*******************************************************************************/
private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName)
{
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
BulkInsertExtractStep.class,
BulkInsertTransformStep.class,
LoadViaInsertStep.class,
values
)
.withName(processName)
.withLabel(table.getLabel() + " Bulk Insert")
.withTableName(table.getName())
.withIsHidden(true);
List<QFieldMetaData> editableFields = table.getFields().values().stream()
.filter(QFieldMetaData::getIsEditable)
.toList();
@ -307,50 +322,13 @@ public class QInstanceEnricher
.withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
// .withValue("text", "Upload a CSV or XLSX file with the following columns: " + fieldsForHelpText));
.withValue("text", "Upload a CSV file with the following columns: " + fieldsForHelpText));
.withValue("previewText", "file upload instructions")
.withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM));
QBackendStepMetaData receiveFileStep = new QBackendStepMetaData()
.withName("receiveFile")
.withCode(new QCodeReference(BulkInsertReceiveFileStep.class))
.withOutputMetaData(new QFunctionOutputMetaData()
.withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData()
.withName("review")
.withRecordListFields(editableFields)
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below were parsed from your file, and will be inserted if you click Submit."))
.withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows"));
QBackendStepMetaData storeStep = new QBackendStepMetaData()
.withName("storeRecords")
.withCode(new QCodeReference(BulkInsertStoreRecordsStep.class))
.withOutputMetaData(new QFunctionOutputMetaData()
.withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData()
.withName("results")
.withRecordListFields(new ArrayList<>(table.getFields().values()))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below have been inserted."))
.withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows"));
qInstance.addProcess(
new QProcessMetaData()
.withName(processName)
.withLabel(table.getLabel() + " Bulk Insert")
.withTableName(table.getName())
.withIsHidden(true)
.withStepList(List.of(
uploadScreen,
receiveFileStep,
reviewScreen,
storeStep,
resultsScreen
)));
process.addStep(0, uploadScreen);
process.getFrontendStep("review").setRecordListFields(editableFields);
qInstance.addProcess(process);
}
@ -360,6 +338,22 @@ public class QInstanceEnricher
*******************************************************************************/
private void defineTableBulkEdit(QInstance qInstance, QTableMetaData table, String processName)
{
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table.getName());
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE);
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
ExtractViaQueryStep.class,
BulkEditTransformStep.class,
LoadViaUpdateStep.class,
values
)
.withName(processName)
.withLabel(table.getLabel() + " Bulk Edit")
.withTableName(table.getName())
.withIsHidden(true);
List<QFieldMetaData> editableFields = table.getFields().values().stream()
.filter(QFieldMetaData::getIsEditable)
.toList();
@ -375,54 +369,11 @@ public class QInstanceEnricher
The values you supply here will be updated in all of the records you are bulk editing.
You can clear out the value in a field by flipping the switch on for that field and leaving the input field blank.
Fields whose switches are off will not be updated."""))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.BULK_EDIT_FORM)
);
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_EDIT_FORM));
QBackendStepMetaData receiveValuesStep = new QBackendStepMetaData()
.withName("receiveValues")
.withCode(new QCodeReference(BulkEditReceiveValuesStep.class))
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName(table.getName()))
.withField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, QFieldType.STRING))
.withFields(editableFields));
QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData()
.withName("review")
.withRecordListFields(editableFields)
.withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below will be updated if you click Submit."));
QBackendStepMetaData storeStep = new QBackendStepMetaData()
.withName("storeRecords")
.withCode(new QCodeReference(BulkEditStoreRecordsStep.class))
.withOutputMetaData(new QFunctionOutputMetaData()
.withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData()
.withName("results")
.withRecordListFields(new ArrayList<>(table.getFields().values()))
.withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below have been updated."));
qInstance.addProcess(
new QProcessMetaData()
.withName(processName)
.withLabel(table.getLabel() + " Bulk Edit")
.withTableName(table.getName())
.withIsHidden(true)
.withStepList(List.of(
LoadInitialRecordsStep.defineMetaData(table.getName()),
editScreen,
receiveValuesStep,
reviewScreen,
storeStep,
resultsScreen
)));
process.addStep(0, editScreen);
process.getFrontendStep("review").setRecordListFields(editableFields);
qInstance.addProcess(process);
}
@ -432,36 +383,26 @@ public class QInstanceEnricher
*******************************************************************************/
private void defineTableBulkDelete(QInstance qInstance, QTableMetaData table, String processName)
{
QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData()
.withName("review")
.withRecordListFields(new ArrayList<>(table.getFields().values()))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below will be deleted if you click Submit."));
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table.getName());
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_DELETE);
QBackendStepMetaData storeStep = new QBackendStepMetaData()
.withName("delete")
.withCode(new QCodeReference(BulkDeleteStoreStep.class));
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
ExtractViaQueryStep.class,
BulkDeleteTransformStep.class,
LoadViaDeleteStep.class,
values
)
.withName(processName)
.withLabel(table.getLabel() + " Bulk Delete")
.withTableName(table.getName())
.withIsHidden(true);
QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData()
.withName("results")
.withRecordListFields(new ArrayList<>(table.getFields().values()))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("text", "The records below have been deleted."));
List<QFieldMetaData> tableFields = table.getFields().values().stream().toList();
process.getFrontendStep("review").setRecordListFields(tableFields);
qInstance.addProcess(
new QProcessMetaData()
.withName(processName)
.withLabel(table.getLabel() + " Bulk Delete")
.withTableName(table.getName())
.withIsHidden(true)
.withStepList(List.of(
LoadInitialRecordsStep.defineMetaData(table.getName()),
reviewScreen,
storeStep,
resultsScreen
)));
qInstance.addProcess(process);
}
@ -569,6 +510,7 @@ public class QInstanceEnricher
}
/*******************************************************************************
** If a table didn't have any sections, generate "sensible defaults"
*******************************************************************************/

335
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@ -39,10 +40,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.logging.log4j.LogManager;
@ -65,6 +70,8 @@ public class QInstanceValidator
private boolean printWarnings = false;
private List<String> errors = new ArrayList<>();
/*******************************************************************************
@ -96,14 +103,14 @@ public class QInstanceValidator
//////////////////////////////////////////////////////////////////////////
// do the validation checks - a good qInstance has all conditions TRUE! //
//////////////////////////////////////////////////////////////////////////
List<String> errors = new ArrayList<>();
try
{
validateBackends(qInstance, errors);
validateTables(qInstance, errors);
validateProcesses(qInstance, errors);
validateApps(qInstance, errors);
validatePossibleValueSources(qInstance, errors);
validateBackends(qInstance);
validateAutomationProviders(qInstance);
validateTables(qInstance);
validateProcesses(qInstance);
validateApps(qInstance);
validatePossibleValueSources(qInstance);
}
catch(Exception e)
{
@ -123,13 +130,13 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateBackends(QInstance qInstance, List<String> errors)
private void validateBackends(QInstance qInstance)
{
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getBackends()), "At least 1 backend must be defined."))
if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getBackends()), "At least 1 backend must be defined."))
{
qInstance.getBackends().forEach((backendName, backend) ->
{
assertCondition(errors, Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
});
}
}
@ -139,42 +146,59 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateTables(QInstance qInstance, List<String> errors)
private void validateAutomationProviders(QInstance qInstance)
{
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getTables()),
if(qInstance.getAutomationProviders() != null)
{
qInstance.getAutomationProviders().forEach((name, automationProvider) ->
{
assertCondition(Objects.equals(name, automationProvider.getName()), "Inconsistent naming for automationProvider: " + name + "/" + automationProvider.getName() + ".");
assertCondition(automationProvider.getType() != null, "Missing type for automationProvider: " + name);
});
}
}
/*******************************************************************************
**
*******************************************************************************/
private void validateTables(QInstance qInstance)
{
if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()),
"At least 1 table must be defined."))
{
qInstance.getTables().forEach((tableName, table) ->
{
assertCondition(errors, Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + ".");
assertCondition(Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + ".");
validateAppChildHasValidParentAppName(qInstance, errors, table);
validateAppChildHasValidParentAppName(qInstance, table);
////////////////////////////////////////
// validate the backend for the table //
////////////////////////////////////////
if(assertCondition(errors, StringUtils.hasContent(table.getBackendName()),
if(assertCondition(StringUtils.hasContent(table.getBackendName()),
"Missing backend name for table " + tableName + "."))
{
if(CollectionUtils.nullSafeHasContents(qInstance.getBackends()))
{
assertCondition(errors, qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + ".");
assertCondition(qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + ".");
}
}
//////////////////////////////////
// validate fields in the table //
//////////////////////////////////
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(table.getFields()), "At least 1 field must be defined in table " + tableName + "."))
if(assertCondition(CollectionUtils.nullSafeHasContents(table.getFields()), "At least 1 field must be defined in table " + tableName + "."))
{
table.getFields().forEach((fieldName, field) ->
{
assertCondition(errors, Objects.equals(fieldName, field.getName()),
assertCondition(Objects.equals(fieldName, field.getName()),
"Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
if(field.getPossibleValueSourceName() != null)
{
assertCondition(errors, qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null,
assertCondition(qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null,
"Unrecognized possibleValueSourceName " + field.getPossibleValueSourceName() + " in table " + tableName + " for field " + fieldName + ".");
}
});
@ -189,10 +213,10 @@ public class QInstanceValidator
{
for(QFieldSection section : table.getSections())
{
validateSection(errors, table, section, fieldNamesInSections);
validateSection(table, section, fieldNamesInSections);
if(section.getTier().equals(Tier.T1))
{
assertCondition(errors, tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1");
assertCondition(tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1");
tier1Section = section;
}
}
@ -202,7 +226,7 @@ public class QInstanceValidator
{
for(String fieldName : table.getFields().keySet())
{
assertCondition(errors, fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections.");
assertCondition(fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections.");
}
}
@ -213,7 +237,7 @@ public class QInstanceValidator
{
for(String recordLabelField : table.getRecordLabelFields())
{
assertCondition(errors, table.getFields().containsKey(recordLabelField), "Table " + tableName + " record label field " + recordLabelField + " is not a field on this table.");
assertCondition(table.getFields().containsKey(recordLabelField), "Table " + tableName + " record label field " + recordLabelField + " is not a field on this table.");
}
}
@ -221,9 +245,17 @@ public class QInstanceValidator
{
for(Map.Entry<String, QCodeReference> entry : table.getCustomizers().entrySet())
{
validateTableCustomizer(errors, tableName, entry.getKey(), entry.getValue());
validateTableCustomizer(tableName, entry.getKey(), entry.getValue());
}
}
//////////////////////////////////////
// validate the table's automations //
//////////////////////////////////////
if(table.getAutomationDetails() != null)
{
validateTableAutomationDetails(qInstance, table);
}
});
}
}
@ -233,11 +265,115 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateTableCustomizer(List<String> errors, String tableName, String customizerName, QCodeReference codeReference)
private void validateTableAutomationDetails(QInstance qInstance, QTableMetaData table)
{
String tableName = table.getName();
String prefix = "Table " + tableName + " automationDetails ";
QTableAutomationDetails automationDetails = table.getAutomationDetails();
//////////////////////////////////////
// validate the automation provider //
//////////////////////////////////////
String providerName = automationDetails.getProviderName();
if(assertCondition(StringUtils.hasContent(providerName), prefix + " is missing a providerName"))
{
assertCondition(qInstance.getAutomationProvider(providerName) != null, " has an unrecognized providerName: " + providerName);
}
//////////////////////////////////
// validate the status tracking //
//////////////////////////////////
AutomationStatusTracking statusTracking = automationDetails.getStatusTracking();
if(assertCondition(statusTracking != null, prefix + "do not have statusTracking defined."))
{
if(assertCondition(statusTracking.getType() != null, prefix + "statusTracking is missing a type"))
{
if(statusTracking.getType().equals(AutomationStatusTrackingType.FIELD_IN_TABLE))
{
if(assertCondition(StringUtils.hasContent(statusTracking.getFieldName()), prefix + "statusTracking of type fieldInTable is missing its fieldName"))
{
assertNoException(() -> table.getField(statusTracking.getFieldName()), prefix + "statusTracking field is not a defined field on this table.");
}
}
}
}
//////////////////////////
// validate the actions //
//////////////////////////
Set<String> usedNames = new HashSet<>();
if(automationDetails.getActions() != null)
{
automationDetails.getActions().forEach(action ->
{
assertCondition(StringUtils.hasContent(action.getName()), prefix + "has an action missing a name");
assertCondition(!usedNames.contains(action.getName()), prefix + "has more than one action named " + action.getName());
usedNames.add(action.getName());
String actionPrefix = prefix + "action " + action.getName() + " ";
assertCondition(action.getTriggerEvent() != null, actionPrefix + "is missing a triggerEvent");
/////////////////////////////////////////////////////
// validate the code or process used by the action //
/////////////////////////////////////////////////////
int numberSet = 0;
if(action.getCodeReference() != null)
{
numberSet++;
if(preAssertionsForCodeReference(action.getCodeReference(), actionPrefix))
{
validateSimpleCodeReference(actionPrefix + "code reference: ", action.getCodeReference(), RecordAutomationHandler.class);
}
}
if(action.getProcessName() != null)
{
numberSet++;
QProcessMetaData process = qInstance.getProcess(action.getProcessName());
if(assertCondition(process != null, actionPrefix + "has an unrecognized processName: " + action.getProcessName()))
{
if(process.getTableName() != null)
{
assertCondition(tableName.equals(process.getTableName()), actionPrefix + " references a process from a different table");
}
}
}
assertCondition(numberSet != 0, actionPrefix + "is missing both a codeReference and a processName");
assertCondition(!(numberSet > 1), actionPrefix + "has both a codeReference and a processName (which is not allowed)");
///////////////////////////////////////////
// validate the filter (if there is one) //
///////////////////////////////////////////
if(action.getFilter() != null && action.getFilter().getCriteria() != null)
{
action.getFilter().getCriteria().forEach((criteria) ->
{
if(assertCondition(StringUtils.hasContent(criteria.getFieldName()), actionPrefix + "has a filter criteria without a field name"))
{
assertNoException(() -> table.getField(criteria.getFieldName()), actionPrefix + "has a filter criteria referencing an unrecognized field: " + criteria.getFieldName());
}
assertCondition(criteria.getOperator() != null, actionPrefix + "has a filter criteria without an operator");
// todo - validate cardinality of values...
});
}
});
}
}
/*******************************************************************************
**
*******************************************************************************/
private void validateTableCustomizer(String tableName, String customizerName, QCodeReference codeReference)
{
String prefix = "Table " + tableName + ", customizer " + customizerName + ": ";
if(!preAssertionsForCodeReference(errors, codeReference, prefix))
if(!preAssertionsForCodeReference(codeReference, prefix))
{
return;
}
@ -245,18 +381,18 @@ public class QInstanceValidator
//////////////////////////////////////////////////////////////////////////////
// make sure (at this time) that it's a java type, then do some java checks //
//////////////////////////////////////////////////////////////////////////////
if(assertCondition(errors, codeReference.getCodeType().equals(QCodeType.JAVA), prefix + "Only JAVA customizers are supported at this time."))
if(assertCondition(codeReference.getCodeType().equals(QCodeType.JAVA), prefix + "Only JAVA customizers are supported at this time."))
{
///////////////////////////////////////
// make sure the class can be loaded //
///////////////////////////////////////
Class<?> customizerClass = getClassForCodeReference(errors, codeReference, prefix);
Class<?> customizerClass = getClassForCodeReference(codeReference, prefix);
if(customizerClass != null)
{
//////////////////////////////////////////////////
// make sure the customizer can be instantiated //
//////////////////////////////////////////////////
Object customizerInstance = getInstanceOfCodeReference(errors, prefix, customizerClass);
Object customizerInstance = getInstanceOfCodeReference(prefix, customizerClass);
TableCustomizers tableCustomizer = TableCustomizers.forRole(customizerName);
if(tableCustomizer == null)
@ -273,7 +409,7 @@ public class QInstanceValidator
////////////////////////////////////////////////////////////////////////
if(customizerInstance != null && tableCustomizer.getTableCustomizer().getExpectedType() != null)
{
Object castedObject = getCastedObject(errors, prefix, tableCustomizer.getTableCustomizer().getExpectedType(), customizerInstance);
Object castedObject = getCastedObject(prefix, tableCustomizer.getTableCustomizer().getExpectedType(), customizerInstance);
Consumer<Object> validationFunction = tableCustomizer.getTableCustomizer().getValidationFunction();
if(castedObject != null && validationFunction != null)
@ -305,7 +441,7 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private <T> T getCastedObject(List<String> errors, String prefix, Class<T> expectedType, Object customizerInstance)
private <T> T getCastedObject(String prefix, Class<T> expectedType, Object customizerInstance)
{
T castedObject = null;
try
@ -314,7 +450,7 @@ public class QInstanceValidator
}
catch(ClassCastException e)
{
errors.add(prefix + "CodeReference could not be casted to the expected type: " + expectedType);
errors.add(prefix + "CodeReference is not of the expected type: " + expectedType);
}
return castedObject;
}
@ -324,7 +460,7 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private Object getInstanceOfCodeReference(List<String> errors, String prefix, Class<?> customizerClass)
private Object getInstanceOfCodeReference(String prefix, Class<?> customizerClass)
{
Object customizerInstance = null;
try
@ -343,18 +479,18 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateSection(List<String> errors, QTableMetaData table, QFieldSection section, Set<String> fieldNamesInSections)
private void validateSection(QTableMetaData table, QFieldSection section, Set<String> fieldNamesInSections)
{
assertCondition(errors, StringUtils.hasContent(section.getName()), "Missing a name for field section in table " + table.getName() + ".");
assertCondition(errors, StringUtils.hasContent(section.getLabel()), "Missing a label for field section in table " + table.getLabel() + ".");
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(section.getFieldNames()), "Table " + table.getName() + " section " + section.getName() + " does not have any fields."))
assertCondition(StringUtils.hasContent(section.getName()), "Missing a name for field section in table " + table.getName() + ".");
assertCondition(StringUtils.hasContent(section.getLabel()), "Missing a label for field section in table " + table.getLabel() + ".");
if(assertCondition(CollectionUtils.nullSafeHasContents(section.getFieldNames()), "Table " + table.getName() + " section " + section.getName() + " does not have any fields."))
{
if(table.getFields() != null)
{
for(String fieldName : section.getFieldNames())
{
assertCondition(errors, table.getFields().containsKey(fieldName), "Table " + table.getName() + " section " + section.getName() + " specifies fieldName " + fieldName + ", which is not a field on this table.");
assertCondition(errors, !fieldNamesInSections.contains(fieldName), "Table " + table.getName() + " has field " + fieldName + " listed more than once in its field sections.");
assertCondition(table.getFields().containsKey(fieldName), "Table " + table.getName() + " section " + section.getName() + " specifies fieldName " + fieldName + ", which is not a field on this table.");
assertCondition(!fieldNamesInSections.contains(fieldName), "Table " + table.getName() + " has field " + fieldName + " listed more than once in its field sections.");
fieldNamesInSections.add(fieldName);
}
@ -367,33 +503,33 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateProcesses(QInstance qInstance, List<String> errors)
private void validateProcesses(QInstance qInstance)
{
if(CollectionUtils.nullSafeHasContents(qInstance.getProcesses()))
{
qInstance.getProcesses().forEach((processName, process) ->
{
assertCondition(errors, Objects.equals(processName, process.getName()), "Inconsistent naming for process: " + processName + "/" + process.getName() + ".");
assertCondition(Objects.equals(processName, process.getName()), "Inconsistent naming for process: " + processName + "/" + process.getName() + ".");
validateAppChildHasValidParentAppName(qInstance, errors, process);
validateAppChildHasValidParentAppName(qInstance, process);
/////////////////////////////////////////////
// validate the table name for the process //
/////////////////////////////////////////////
if(process.getTableName() != null)
{
assertCondition(errors, qInstance.getTable(process.getTableName()) != null, "Unrecognized table " + process.getTableName() + " for process " + processName + ".");
assertCondition(qInstance.getTable(process.getTableName()) != null, "Unrecognized table " + process.getTableName() + " for process " + processName + ".");
}
///////////////////////////////////
// validate steps in the process //
///////////////////////////////////
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(process.getStepList()), "At least 1 step must be defined in process " + processName + "."))
if(assertCondition(CollectionUtils.nullSafeHasContents(process.getStepList()), "At least 1 step must be defined in process " + processName + "."))
{
int index = 0;
for(QStepMetaData step : process.getStepList())
{
assertCondition(errors, StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName);
assertCondition(StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName);
index++;
}
}
@ -406,26 +542,26 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateApps(QInstance qInstance, List<String> errors)
private void validateApps(QInstance qInstance)
{
if(CollectionUtils.nullSafeHasContents(qInstance.getApps()))
{
qInstance.getApps().forEach((appName, app) ->
{
assertCondition(errors, Objects.equals(appName, app.getName()), "Inconsistent naming for app: " + appName + "/" + app.getName() + ".");
assertCondition(Objects.equals(appName, app.getName()), "Inconsistent naming for app: " + appName + "/" + app.getName() + ".");
validateAppChildHasValidParentAppName(qInstance, errors, app);
validateAppChildHasValidParentAppName(qInstance, app);
Set<String> appsVisited = new HashSet<>();
visitAppCheckingForCycles(app, appsVisited, errors);
visitAppCheckingForCycles(app, appsVisited);
if(app.getChildren() != null)
{
Set<String> childNames = new HashSet<>();
for(QAppChildMetaData child : app.getChildren())
{
assertCondition(errors, Objects.equals(appName, child.getParentAppName()), "Child " + child.getName() + " of app " + appName + " does not have its parent app properly set.");
assertCondition(errors, !childNames.contains(child.getName()), "App " + appName + " contains more than one child named " + child.getName());
assertCondition(Objects.equals(appName, child.getParentAppName()), "Child " + child.getName() + " of app " + appName + " does not have its parent app properly set.");
assertCondition(!childNames.contains(child.getName()), "App " + appName + " contains more than one child named " + child.getName());
childNames.add(child.getName());
}
}
@ -438,14 +574,14 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validatePossibleValueSources(QInstance qInstance, List<String> errors)
private void validatePossibleValueSources(QInstance qInstance)
{
if(CollectionUtils.nullSafeHasContents(qInstance.getPossibleValueSources()))
{
qInstance.getPossibleValueSources().forEach((pvsName, possibleValueSource) ->
{
assertCondition(errors, Objects.equals(pvsName, possibleValueSource.getName()), "Inconsistent naming for possibleValueSource: " + pvsName + "/" + possibleValueSource.getName() + ".");
if(assertCondition(errors, possibleValueSource.getType() != null, "Missing type for possibleValueSource: " + pvsName))
assertCondition(Objects.equals(pvsName, possibleValueSource.getName()), "Inconsistent naming for possibleValueSource: " + pvsName + "/" + possibleValueSource.getName() + ".");
if(assertCondition(possibleValueSource.getType() != null, "Missing type for possibleValueSource: " + pvsName))
{
////////////////////////////////////////////////////////////////////////////////////////////////
// assert about fields that should and should not be set, based on possible value source type //
@ -455,30 +591,30 @@ public class QInstanceValidator
{
case ENUM ->
{
assertCondition(errors, !StringUtils.hasContent(possibleValueSource.getTableName()), "enum-type possibleValueSource " + pvsName + " should not have a tableName.");
assertCondition(errors, possibleValueSource.getCustomCodeReference() == null, "enum-type possibleValueSource " + pvsName + " should not have a customCodeReference.");
assertCondition(!StringUtils.hasContent(possibleValueSource.getTableName()), "enum-type possibleValueSource " + pvsName + " should not have a tableName.");
assertCondition(possibleValueSource.getCustomCodeReference() == null, "enum-type possibleValueSource " + pvsName + " should not have a customCodeReference.");
assertCondition(errors, CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues()), "enum-type possibleValueSource " + pvsName + " is missing enum values");
assertCondition(CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues()), "enum-type possibleValueSource " + pvsName + " is missing enum values");
}
case TABLE ->
{
assertCondition(errors, CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "table-type possibleValueSource " + pvsName + " should not have enum values.");
assertCondition(errors, possibleValueSource.getCustomCodeReference() == null, "table-type possibleValueSource " + pvsName + " should not have a customCodeReference.");
assertCondition(CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "table-type possibleValueSource " + pvsName + " should not have enum values.");
assertCondition(possibleValueSource.getCustomCodeReference() == null, "table-type possibleValueSource " + pvsName + " should not have a customCodeReference.");
if(assertCondition(errors, StringUtils.hasContent(possibleValueSource.getTableName()), "table-type possibleValueSource " + pvsName + " is missing a tableName."))
if(assertCondition(StringUtils.hasContent(possibleValueSource.getTableName()), "table-type possibleValueSource " + pvsName + " is missing a tableName."))
{
assertCondition(errors, qInstance.getTable(possibleValueSource.getTableName()) != null, "Unrecognized table " + possibleValueSource.getTableName() + " for possibleValueSource " + pvsName + ".");
assertCondition(qInstance.getTable(possibleValueSource.getTableName()) != null, "Unrecognized table " + possibleValueSource.getTableName() + " for possibleValueSource " + pvsName + ".");
}
}
case CUSTOM ->
{
assertCondition(errors, CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "custom-type possibleValueSource " + pvsName + " should not have enum values.");
assertCondition(errors, !StringUtils.hasContent(possibleValueSource.getTableName()), "custom-type possibleValueSource " + pvsName + " should not have a tableName.");
assertCondition(CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "custom-type possibleValueSource " + pvsName + " should not have enum values.");
assertCondition(!StringUtils.hasContent(possibleValueSource.getTableName()), "custom-type possibleValueSource " + pvsName + " should not have a tableName.");
if(assertCondition(errors, possibleValueSource.getCustomCodeReference() != null, "custom-type possibleValueSource " + pvsName + " is missing a customCodeReference."))
if(assertCondition(possibleValueSource.getCustomCodeReference() != null, "custom-type possibleValueSource " + pvsName + " is missing a customCodeReference."))
{
assertCondition(errors, QCodeUsage.POSSIBLE_VALUE_PROVIDER.equals(possibleValueSource.getCustomCodeReference().getCodeUsage()), "customCodeReference for possibleValueSource " + pvsName + " is not a possibleValueProvider.");
validateCustomPossibleValueSourceCode(errors, pvsName, possibleValueSource.getCustomCodeReference());
assertCondition(QCodeUsage.POSSIBLE_VALUE_PROVIDER.equals(possibleValueSource.getCustomCodeReference().getCodeUsage()), "customCodeReference for possibleValueSource " + pvsName + " is not a possibleValueProvider.");
validateSimpleCodeReference("PossibleValueSource " + pvsName + " custom code reference: ", possibleValueSource.getCustomCodeReference(), QCustomPossibleValueProvider.class);
}
}
default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType());
@ -493,11 +629,9 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateCustomPossibleValueSourceCode(List<String> errors, String pvsName, QCodeReference codeReference)
private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class<?> expectedClass)
{
String prefix = "PossibleValueSource " + pvsName + " custom code reference: ";
if(!preAssertionsForCodeReference(errors, codeReference, prefix))
if(!preAssertionsForCodeReference(codeReference, prefix))
{
return;
}
@ -505,25 +639,25 @@ public class QInstanceValidator
//////////////////////////////////////////////////////////////////////////////
// make sure (at this time) that it's a java type, then do some java checks //
//////////////////////////////////////////////////////////////////////////////
if(assertCondition(errors, codeReference.getCodeType().equals(QCodeType.JAVA), prefix + "Only JAVA customizers are supported at this time."))
if(assertCondition(codeReference.getCodeType().equals(QCodeType.JAVA), prefix + "Only JAVA code references are supported at this time."))
{
///////////////////////////////////////
// make sure the class can be loaded //
///////////////////////////////////////
Class<?> customizerClass = getClassForCodeReference(errors, codeReference, prefix);
Class<?> customizerClass = getClassForCodeReference(codeReference, prefix);
if(customizerClass != null)
{
//////////////////////////////////////////////////
// make sure the customizer can be instantiated //
//////////////////////////////////////////////////
Object customizerInstance = getInstanceOfCodeReference(errors, prefix, customizerClass);
Object customizerInstance = getInstanceOfCodeReference(prefix, customizerClass);
////////////////////////////////////////////////////////////////////////
// make sure the customizer instance can be cast to the expected type //
////////////////////////////////////////////////////////////////////////
if(customizerInstance != null)
{
getCastedObject(errors, prefix, QCustomPossibleValueProvider.class, customizerInstance);
getCastedObject(prefix, expectedClass, customizerInstance);
}
}
}
@ -534,7 +668,7 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private Class<?> getClassForCodeReference(List<String> errors, QCodeReference codeReference, String prefix)
private Class<?> getClassForCodeReference(QCodeReference codeReference, String prefix)
{
Class<?> customizerClass = null;
try
@ -553,15 +687,15 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private boolean preAssertionsForCodeReference(List<String> errors, QCodeReference codeReference, String prefix)
private boolean preAssertionsForCodeReference(QCodeReference codeReference, String prefix)
{
boolean okay = true;
if(!assertCondition(errors, StringUtils.hasContent(codeReference.getName()), prefix + " is missing a code reference name"))
if(!assertCondition(StringUtils.hasContent(codeReference.getName()), prefix + " is missing a code reference name"))
{
okay = false;
}
if(!assertCondition(errors, codeReference.getCodeType() != null, prefix + " is missing a code type"))
if(!assertCondition(codeReference.getCodeType() != null, prefix + " is missing a code type"))
{
okay = false;
}
@ -575,9 +709,9 @@ public class QInstanceValidator
** Check if an app's child list can recursively be traversed without finding a
** duplicate, which would indicate a cycle (e.g., an error)
*******************************************************************************/
private void visitAppCheckingForCycles(QAppMetaData app, Set<String> appsVisited, List<String> errors)
private void visitAppCheckingForCycles(QAppMetaData app, Set<String> appsVisited)
{
if(assertCondition(errors, !appsVisited.contains(app.getName()), "Circular app reference detected, involving " + app.getName()))
if(assertCondition(!appsVisited.contains(app.getName()), "Circular app reference detected, involving " + app.getName()))
{
appsVisited.add(app.getName());
if(app.getChildren() != null)
@ -586,7 +720,7 @@ public class QInstanceValidator
{
if(child instanceof QAppMetaData childApp)
{
visitAppCheckingForCycles(childApp, appsVisited, errors);
visitAppCheckingForCycles(childApp, appsVisited);
}
}
}
@ -598,11 +732,11 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateAppChildHasValidParentAppName(QInstance qInstance, List<String> errors, QAppChildMetaData appChild)
private void validateAppChildHasValidParentAppName(QInstance qInstance, QAppChildMetaData appChild)
{
if(appChild.getParentAppName() != null)
{
assertCondition(errors, qInstance.getApp(appChild.getParentAppName()) != null, "Unrecognized parent app " + appChild.getParentAppName() + " for " + appChild.getName() + ".");
assertCondition(qInstance.getApp(appChild.getParentAppName()) != null, "Unrecognized parent app " + appChild.getParentAppName() + " for " + appChild.getName() + ".");
}
}
@ -613,7 +747,7 @@ public class QInstanceValidator
** But if it's false, add the provided message to the list of errors (and return false,
** e.g., in case you need to stop evaluating rules to avoid exceptions).
*******************************************************************************/
private boolean assertCondition(List<String> errors, boolean condition, String message)
private boolean assertCondition(boolean condition, String message)
{
if(!condition)
{
@ -625,6 +759,41 @@ public class QInstanceValidator
/*******************************************************************************
** For the given lambda, if it doesn't throw an exception, then we're all good (and return true).
** But if it throws, add the provided message to the list of errors (and return false,
** e.g., in case you need to stop evaluating rules to avoid exceptions).
*******************************************************************************/
private boolean assertNoException(UnsafeLambda unsafeLambda, String message)
{
try
{
unsafeLambda.run();
return (true);
}
catch(Exception e)
{
errors.add(message);
return (false);
}
}
/*******************************************************************************
**
*******************************************************************************/
@FunctionalInterface
interface UnsafeLambda
{
/*******************************************************************************
**
*******************************************************************************/
void run() throws Exception;
}
/*******************************************************************************
**
*******************************************************************************/

38
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java
View File

@ -23,9 +23,12 @@ package com.kingsrook.qqq.backend.core.instances;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import io.github.cdimascio.dotenv.Dotenv;
import io.github.cdimascio.dotenv.DotenvEntry;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -45,7 +48,22 @@ public class QMetaDataVariableInterpreter
{
private static final Logger LOG = LogManager.getLogger(QMetaDataVariableInterpreter.class);
private Map<String, String> customEnvironment;
private Map<String, String> environmentOverrides;
/*******************************************************************************
**
*******************************************************************************/
public QMetaDataVariableInterpreter()
{
environmentOverrides = new HashMap<>();
Dotenv dotenv = Dotenv.configure().ignoreIfMissing().load();
for(DotenvEntry e : dotenv.entries())
{
environmentOverrides.put(e.getKey(), e.getValue());
}
}
@ -86,7 +104,7 @@ public class QMetaDataVariableInterpreter
//////////////////////////////////////////////////////////////////////////////////////////////
// get the value - if it's null, move on, else, interpret it, and put it back in the object //
//////////////////////////////////////////////////////////////////////////////////////////////
Object value = getter.invoke(o);
Object value = getter.invoke(o);
if(value == null)
{
continue;
@ -124,7 +142,7 @@ public class QMetaDataVariableInterpreter
if(value.startsWith(envPrefix) && value.endsWith("}"))
{
String envVarName = value.substring(envPrefix.length()).replaceFirst("}$", "");
String envValue = getEnvironment().get(envVarName);
String envValue = getEnvironmentVariable(envVarName);
return (envValue);
}
@ -149,13 +167,13 @@ public class QMetaDataVariableInterpreter
/*******************************************************************************
** Setter for customEnvironment - protected - meant to be called (at least at this
** Setter for environmentOverrides - protected - meant to be called (at least at this
** time), only in unit test
**
*******************************************************************************/
protected void setCustomEnvironment(Map<String, String> customEnvironment)
protected void setEnvironmentOverrides(Map<String, String> environmentOverrides)
{
this.customEnvironment = customEnvironment;
this.environmentOverrides = environmentOverrides;
}
@ -163,13 +181,13 @@ public class QMetaDataVariableInterpreter
/*******************************************************************************
**
*******************************************************************************/
private Map<String, String> getEnvironment()
private String getEnvironmentVariable(String key)
{
if(this.customEnvironment != null)
if(this.environmentOverrides.containsKey(key))
{
return (this.customEnvironment);
return (this.environmentOverrides.get(key));
}
return System.getenv();
return System.getenv(key);
}
}

26
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.metadata;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMetaData;
@ -43,6 +44,8 @@ public class MetaDataOutput extends AbstractActionOutput
private List<AppTreeNode> appTree;
private QBrandingMetaData branding;
/*******************************************************************************
@ -89,7 +92,6 @@ public class MetaDataOutput extends AbstractActionOutput
/*******************************************************************************
** Getter for appTree
**
@ -131,4 +133,26 @@ public class MetaDataOutput extends AbstractActionOutput
{
this.apps = apps;
}
/*******************************************************************************
** Getter for branding
**
*******************************************************************************/
public QBrandingMetaData getBranding()
{
return branding;
}
/*******************************************************************************
** Setter for branding
**
*******************************************************************************/
public void setBranding(QBrandingMetaData branding)
{
this.branding = branding;
}
}

22
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java
View File

@ -38,6 +38,7 @@ public class ProcessState implements Serializable
{
private List<QRecord> records = new ArrayList<>();
private Map<String, Serializable> values = new HashMap<>();
private List<String> stepList = new ArrayList<>();
private Optional<String> nextStepName = Optional.empty();
@ -117,4 +118,25 @@ public class ProcessState implements Serializable
this.nextStepName = Optional.empty();
}
/*******************************************************************************
** Getter for stepList
**
*******************************************************************************/
public List<String> getStepList()
{
return stepList;
}
/*******************************************************************************
** Setter for stepList
**
*******************************************************************************/
public void setStepList(List<String> stepList)
{
this.stepList = stepList;
}
}

243
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java Normal file
View File

@ -0,0 +1,243 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.processes;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
/*******************************************************************************
** For processes that may show a review & result screen, this class provides a
** standard way to summarize information about the records in the process.
**
*******************************************************************************/
public class ProcessSummaryLine implements Serializable
{
private Status status;
private Integer count = 0;
private String message;
//////////////////////////////////////////////////////////////////////////
// using ArrayList, because we need to be Serializable, and List is not //
//////////////////////////////////////////////////////////////////////////
private ArrayList<Serializable> primaryKeys;
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLine(Status status, Integer count, String message, ArrayList<Serializable> primaryKeys)
{
this.status = status;
this.count = count;
this.message = message;
this.primaryKeys = primaryKeys;
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLine(Status status, Integer count, String message)
{
this.status = status;
this.count = count;
this.message = message;
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLine(Status status, String message)
{
this.status = status;
this.message = message;
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLine(Status status)
{
this.status = status;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return "ProcessSummaryLine{status=" + status + ", count=" + count + ", message='" + message + '\'' + '}';
}
/*******************************************************************************
** Getter for status
**
*******************************************************************************/
public Status getStatus()
{
return status;
}
/*******************************************************************************
** Setter for status
**
*******************************************************************************/
public void setStatus(Status status)
{
this.status = status;
}
/*******************************************************************************
** Getter for primaryKeys
**
*******************************************************************************/
public List<Serializable> getPrimaryKeys()
{
return primaryKeys;
}
/*******************************************************************************
** Setter for primaryKeys
**
*******************************************************************************/
public void setPrimaryKeys(ArrayList<Serializable> primaryKeys)
{
this.primaryKeys = primaryKeys;
}
/*******************************************************************************
** Getter for count
**
*******************************************************************************/
public Integer getCount()
{
return count;
}
/*******************************************************************************
** Setter for count
**
*******************************************************************************/
public void setCount(Integer count)
{
this.count = count;
}
/*******************************************************************************
** Getter for message
**
*******************************************************************************/
public String getMessage()
{
return message;
}
/*******************************************************************************
** Setter for message
**
*******************************************************************************/
public void setMessage(String message)
{
this.message = message;
}
/*******************************************************************************
**
*******************************************************************************/
public void incrementCount()
{
incrementCount(1);
}
/*******************************************************************************
**
*******************************************************************************/
public void incrementCount(int amount)
{
if(count == null)
{
count = 0;
}
count += amount;
}
/*******************************************************************************
**
*******************************************************************************/
public void incrementCountAndAddPrimaryKey(Serializable primaryKey)
{
incrementCount();
if(primaryKeys == null)
{
primaryKeys = new ArrayList<>();
}
primaryKeys.add(primaryKey);
}
/*******************************************************************************
**
*******************************************************************************/
public void addSelfToListIfAnyCount(ArrayList<ProcessSummaryLine> rs)
{
if(count != null && count > 0)
{
rs.add(this);
}
}
}

49
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java
View File

@ -51,6 +51,10 @@ public class RunBackendStepInput extends AbstractActionInput
private QProcessCallback callback;
private AsyncJobCallback asyncJobCallback;
////////////////////////////////////////////////////////////////////////////
// note - new fields should generally be added in method: cloneFieldsInto //
////////////////////////////////////////////////////////////////////////////
/*******************************************************************************
@ -85,6 +89,25 @@ public class RunBackendStepInput extends AbstractActionInput
/*******************************************************************************
** Kinda like a reverse copy-constructor -- for a subclass that wants all the
** field values from this object. Keep this in sync with the fields in this class!
**
** Of note - the processState does NOT get cloned - because... well, in our first
** use-case (a subclass that doesn't WANT the same/full state), that's what we needed.
*******************************************************************************/
public void cloneFieldsInto(RunBackendStepInput target)
{
target.setStepName(getStepName());
target.setSession(getSession());
target.setTableName(getTableName());
target.setProcessName(getProcessName());
target.setAsyncJobCallback(getAsyncJobCallback());
target.setValues(getValues());
}
/*******************************************************************************
**
*******************************************************************************/
@ -354,7 +377,30 @@ public class RunBackendStepInput extends AbstractActionInput
*******************************************************************************/
public String getValueString(String fieldName)
{
return ((String) getValue(fieldName));
return (ValueUtils.getValueAsString(getValue(fieldName)));
}
/*******************************************************************************
** Getter for a single field's value
**
*******************************************************************************/
public Boolean getValueBoolean(String fieldName)
{
return (ValueUtils.getValueAsBoolean(getValue(fieldName)));
}
/*******************************************************************************
** Getter for a single field's value as a primitive boolean - with null => false.
**
*******************************************************************************/
public boolean getValuePrimitiveBoolean(String fieldName)
{
Boolean valueAsBoolean = ValueUtils.getValueAsBoolean(getValue(fieldName));
return (valueAsBoolean != null && valueAsBoolean);
}
@ -406,4 +452,5 @@ public class RunBackendStepInput extends AbstractActionInput
}
return (asyncJobCallback);
}
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java Normal file
View File

@ -0,0 +1,34 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.processes;
/*******************************************************************************
** Simple status enum - initially for statuses in process status lines.
*******************************************************************************/
public enum Status
{
OK,
WARNING,
ERROR,
INFO
}

42
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.delete;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -35,8 +36,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
*******************************************************************************/
public class DeleteInput extends AbstractTableActionInput
{
private List<Serializable> primaryKeys;
private QQueryFilter queryFilter;
private QBackendTransaction transaction;
private List<Serializable> primaryKeys;
private QQueryFilter queryFilter;
@ -59,6 +61,40 @@ public class DeleteInput extends AbstractTableActionInput
/*******************************************************************************
** Getter for transaction
**
*******************************************************************************/
public QBackendTransaction getTransaction()
{
return transaction;
}
/*******************************************************************************
** Setter for transaction
**
*******************************************************************************/
public void setTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
}
/*******************************************************************************
** Fluent setter for transaction
**
*******************************************************************************/
public DeleteInput withTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
return (this);
}
/*******************************************************************************
** Getter for ids
**
@ -92,6 +128,7 @@ public class DeleteInput extends AbstractTableActionInput
}
/*******************************************************************************
** Getter for queryFilter
**
@ -113,6 +150,7 @@ public class DeleteInput extends AbstractTableActionInput
}
/*******************************************************************************
** Fluent setter for queryFilter
**

38
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.update;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -34,7 +35,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
*******************************************************************************/
public class UpdateInput extends AbstractTableActionInput
{
private List<QRecord> records;
private QBackendTransaction transaction;
private List<QRecord> records;
////////////////////////////////////////////////////////////////////////////////////////////
// allow a caller to specify that they KNOW this optimization (e.g., in SQL) can be made. //
@ -65,6 +67,40 @@ public class UpdateInput extends AbstractTableActionInput
/*******************************************************************************
** Getter for transaction
**
*******************************************************************************/
public QBackendTransaction getTransaction()
{
return transaction;
}
/*******************************************************************************
** Setter for transaction
**
*******************************************************************************/
public void setTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
}
/*******************************************************************************
** Fluent setter for transaction
**
*******************************************************************************/
public UpdateInput withTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
return (this);
}
/*******************************************************************************
** Getter for records
**

118
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java Normal file
View File

@ -0,0 +1,118 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.automation;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
/*******************************************************************************
** Input data for the RecordAutomationHandler interface.
*******************************************************************************/
public class RecordAutomationInput extends AbstractTableActionInput
{
private TableAutomationAction action;
private List<QRecord> recordList;
/*******************************************************************************
**
*******************************************************************************/
public RecordAutomationInput(QInstance instance)
{
super(instance);
}
/*******************************************************************************
** Getter for action
**
*******************************************************************************/
public TableAutomationAction getAction()
{
return action;
}
/*******************************************************************************
** Setter for action
**
*******************************************************************************/
public void setAction(TableAutomationAction action)
{
this.action = action;
}
/*******************************************************************************
** Fluent setter for action
**
*******************************************************************************/
public RecordAutomationInput withAction(TableAutomationAction action)
{
this.action = action;
return (this);
}
/*******************************************************************************
** Getter for recordList
**
*******************************************************************************/
public List<QRecord> getRecordList()
{
return recordList;
}
/*******************************************************************************
** Setter for recordList
**
*******************************************************************************/
public void setRecordList(List<QRecord> recordList)
{
this.recordList = recordList;
}
/*******************************************************************************
** Fluent setter for recordList
**
*******************************************************************************/
public RecordAutomationInput withRecordList(List<QRecord> recordList)
{
this.recordList = recordList;
return (this);
}
}

298
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java Normal file
View File

@ -0,0 +1,298 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
import java.util.List;
/*******************************************************************************
** Model containing datastructure expected by frontend bar chart widget
**
*******************************************************************************/
public class BarChart implements QWidget
{
/*
type: "barChart",
title: "Parcel Invoice Lines per Month",
barChartData: {
labels: ["Feb 22", "Mar 22", "Apr 22", "May 22", "Jun 22", "Jul 22", "Aug 22"],
datasets: {label: "Parcel Invoice Lines", data: [50000, 22000, 11111, 22333, 40404, 9876, 2355]},
},
*/
private String title;
private Data barChartData;
/*******************************************************************************
**
*******************************************************************************/
public BarChart(String title, String seriesLabel, List<String> labels, List<Number> data)
{
setTitle(title);
setBarChartData(new BarChart.Data()
.withLabels(labels)
.withDatasets(new BarChart.Data.DataSet()
.withLabel(seriesLabel)
.withData(data)));
}
/*******************************************************************************
** Getter for type
**
*******************************************************************************/
public String getType()
{
return "barChart";
}
/*******************************************************************************
** Getter for title
**
*******************************************************************************/
public String getTitle()
{
return title;
}
/*******************************************************************************
** Setter for title
**
*******************************************************************************/
public void setTitle(String title)
{
this.title = title;
}
/*******************************************************************************
** Fluent setter for title
**
*******************************************************************************/
public BarChart withTitle(String title)
{
this.title = title;
return (this);
}
/*******************************************************************************
** Getter for barChartData
**
*******************************************************************************/
public Data getBarChartData()
{
return barChartData;
}
/*******************************************************************************
** Setter for barChartData
**
*******************************************************************************/
public void setBarChartData(Data barChartData)
{
this.barChartData = barChartData;
}
/*******************************************************************************
** Fluent setter for barChartData
**
*******************************************************************************/
public BarChart withBarChartData(Data barChartData)
{
this.barChartData = barChartData;
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public static class Data
{
private List<String> labels;
private DataSet datasets;
/*******************************************************************************
** Getter for labels
**
*******************************************************************************/
public List<String> getLabels()
{
return labels;
}
/*******************************************************************************
** Setter for labels
**
*******************************************************************************/
public void setLabels(List<String> labels)
{
this.labels = labels;
}
/*******************************************************************************
** Fluent setter for labels
**
*******************************************************************************/
public Data withLabels(List<String> labels)
{
this.labels = labels;
return (this);
}
/*******************************************************************************
** Getter for datasets
**
*******************************************************************************/
public DataSet getDatasets()
{
return datasets;
}
/*******************************************************************************
** Setter for datasets
**
*******************************************************************************/
public void setDatasets(DataSet datasets)
{
this.datasets = datasets;
}
/*******************************************************************************
** Fluent setter for datasets
**
*******************************************************************************/
public Data withDatasets(DataSet datasets)
{
this.datasets = datasets;
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public static class DataSet
{
private String label;
private List<Number> data;
/*******************************************************************************
** Getter for label
**
*******************************************************************************/
public String getLabel()
{
return label;
}
/*******************************************************************************
** Setter for label
**
*******************************************************************************/
public void setLabel(String label)
{
this.label = label;
}
/*******************************************************************************
** Fluent setter for label
**
*******************************************************************************/
public DataSet withLabel(String label)
{
this.label = label;
return (this);
}
/*******************************************************************************
** Getter for data
**
*******************************************************************************/
public List<Number> getData()
{
return data;
}
/*******************************************************************************
** Setter for data
**
*******************************************************************************/
public void setData(List<Number> data)
{
this.data = data;
}
/*******************************************************************************
** Fluent setter for data
**
*******************************************************************************/
public DataSet withData(List<Number> data)
{
this.data = data;
return (this);
}
}
}
}

36
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java Normal file
View File

@ -0,0 +1,36 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
/*******************************************************************************
** Interface for frontend widget's datastructures
**
*******************************************************************************/
public interface QWidget
{
/*******************************************************************************
** Getter for type
*******************************************************************************/
String getType();
}

162
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java Normal file
View File

@ -0,0 +1,162 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
/*******************************************************************************
** Model containing datastructure expected by frontend AWS quick sight widget
** TODO: this might just be an IFrameChart widget in the future
**
*******************************************************************************/
public class QuickSightChart implements QWidget
{
private String label;
private String name;
private String url;
/*******************************************************************************
**
*******************************************************************************/
public QuickSightChart(String name, String label, String url)
{
this.url = url;
this.name = name;
this.label = label;
}
/*******************************************************************************
** Getter for type
**
*******************************************************************************/
public String getType()
{
return "quickSightChart";
}
/*******************************************************************************
** Getter for url
**
*******************************************************************************/
public String getUrl()
{
return url;
}
/*******************************************************************************
** Setter for url
**
*******************************************************************************/
public void setUrl(String url)
{
this.url = url;
}
/*******************************************************************************
** Fluent setter for url
**
*******************************************************************************/
public QuickSightChart withUrl(String url)
{
this.url = url;
return (this);
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public QuickSightChart withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for label
**
*******************************************************************************/
public String getLabel()
{
return label;
}
/*******************************************************************************
** Setter for label
**
*******************************************************************************/
public void setLabel(String label)
{
this.label = label;
}
/*******************************************************************************
** Fluent setter for label
**
*******************************************************************************/
public QuickSightChart withLabel(String label)
{
this.label = label;
return (this);
}
}

67
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java
View File

@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang.SerializationUtils;
/*******************************************************************************
@ -87,16 +88,70 @@ public class QRecord implements Serializable
/*******************************************************************************
** Copy constructor.
** TODO ... should this do deep copies?
**
*******************************************************************************/
public QRecord(QRecord record)
{
this.tableName = record.tableName;
this.recordLabel = record.recordLabel;
this.values = record.values;
this.displayValues = record.displayValues;
this.backendDetails = record.backendDetails;
this.errors = record.errors;
this.values = doDeepCopy(record.values);
this.displayValues = doDeepCopy(record.displayValues);
this.backendDetails = doDeepCopy(record.backendDetails);
this.errors = doDeepCopy(record.errors);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return "QRecord{tableName='" + tableName + "',id='" + getValue("id") + "'}";
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings({ "unchecked" })
private <K, V> Map<K, V> doDeepCopy(Map<K, V> map)
{
if(map == null)
{
return (null);
}
if(map instanceof Serializable serializableMap)
{
return (Map<K, V>) SerializationUtils.clone(serializableMap);
}
return (new LinkedHashMap<>(map));
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings({ "unchecked" })
private <T> List<T> doDeepCopy(List<T> list)
{
if(list == null)
{
return (null);
}
if(list instanceof Serializable serializableList)
{
return (List<T>) SerializationUtils.clone(serializableList);
}
return (new ArrayList<>(list));
}
@ -142,7 +197,6 @@ public class QRecord implements Serializable
/*******************************************************************************
**
*******************************************************************************/
@ -209,6 +263,7 @@ public class QRecord implements Serializable
}
/*******************************************************************************
** Fluent setter for recordLabel
**

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QAuthenticationType.java
View File

@ -26,7 +26,6 @@ package com.kingsrook.qqq.backend.core.model.metadata;
** Enum to define the possible authentication types
**
*******************************************************************************/
@SuppressWarnings("rawtypes")
public enum QAuthenticationType
{
AUTH_0("auth0"),

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java
View File

@ -75,11 +75,10 @@ public class QBackendMetaData
/*******************************************************************************
** Fluent setter, returning generically, to help sub-class fluent flows
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends QBackendMetaData> T withName(String name)
public QBackendMetaData withName(String name)
{
this.name = name;
return (T) this;
return this;
}

387
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java
View File

@ -29,12 +29,16 @@ import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey;
import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
@ -49,7 +53,9 @@ public class QInstance
@JsonIgnore
private Map<String, QBackendMetaData> backends = new HashMap<>();
private QAuthenticationMetaData authentication = null;
private QAuthenticationMetaData authentication = null;
private QBrandingMetaData branding = null;
private Map<String, QAutomationProviderMetaData> automationProviders = new HashMap<>();
////////////////////////////////////////////////////////////////////////////////////////////
// Important to use LinkedHashmap here, to preserve the order in which entries are added. //
@ -59,6 +65,8 @@ public class QInstance
private Map<String, QProcessMetaData> processes = new LinkedHashMap<>();
private Map<String, QAppMetaData> apps = new LinkedHashMap<>();
private Map<String, QWidgetMetaDataInterface> widgets = new LinkedHashMap<>();
// todo - lock down the object (no more changes allowed) after it's been validated?
@JsonIgnore
@ -103,17 +111,6 @@ public class QInstance
/*******************************************************************************
** Setter for hasBeenValidated
**
*******************************************************************************/
public void setHasBeenValidated(QInstanceValidationKey key)
{
this.hasBeenValidated = true;
}
/*******************************************************************************
**
*******************************************************************************/
@ -129,6 +126,10 @@ public class QInstance
*******************************************************************************/
public void addBackend(String name, QBackendMetaData backend)
{
if(!StringUtils.hasContent(name))
{
throw (new IllegalArgumentException("Attempted to add a backend without a name."));
}
if(this.backends.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second backend with name: " + name));
@ -148,6 +149,28 @@ public class QInstance
/*******************************************************************************
** Getter for backends
**
*******************************************************************************/
public Map<String, QBackendMetaData> getBackends()
{
return backends;
}
/*******************************************************************************
** Setter for backends
**
*******************************************************************************/
public void setBackends(Map<String, QBackendMetaData> backends)
{
this.backends = backends;
}
/*******************************************************************************
**
*******************************************************************************/
@ -163,6 +186,10 @@ public class QInstance
*******************************************************************************/
public void addTable(String name, QTableMetaData table)
{
if(!StringUtils.hasContent(name))
{
throw (new IllegalArgumentException("Attempted to add a table without a name."));
}
if(this.tables.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second table with name: " + name));
@ -187,6 +214,28 @@ public class QInstance
/*******************************************************************************
** Getter for tables
**
*******************************************************************************/
public Map<String, QTableMetaData> getTables()
{
return tables;
}
/*******************************************************************************
** Setter for tables
**
*******************************************************************************/
public void setTables(Map<String, QTableMetaData> tables)
{
this.tables = tables;
}
/*******************************************************************************
**
*******************************************************************************/
@ -202,6 +251,10 @@ public class QInstance
*******************************************************************************/
public void addPossibleValueSource(String name, QPossibleValueSource possibleValueSource)
{
if(!StringUtils.hasContent(name))
{
throw (new IllegalArgumentException("Attempted to add a possibleValueSource without a name."));
}
if(this.possibleValueSources.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second possibleValueSource with name: " + name));
@ -221,6 +274,28 @@ public class QInstance
/*******************************************************************************
** Getter for possibleValueSources
**
*******************************************************************************/
public Map<String, QPossibleValueSource> getPossibleValueSources()
{
return possibleValueSources;
}
/*******************************************************************************
** Setter for possibleValueSources
**
*******************************************************************************/
public void setPossibleValueSources(Map<String, QPossibleValueSource> possibleValueSources)
{
this.possibleValueSources = possibleValueSources;
}
/*******************************************************************************
**
*******************************************************************************/
@ -252,6 +327,10 @@ public class QInstance
*******************************************************************************/
public void addProcess(String name, QProcessMetaData process)
{
if(!StringUtils.hasContent(name))
{
throw (new IllegalArgumentException("Attempted to add a process without a name."));
}
if(this.processes.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second process with name: " + name));
@ -271,106 +350,6 @@ public class QInstance
/*******************************************************************************
**
*******************************************************************************/
public void addApp(QAppMetaData app)
{
this.addApp(app.getName(), app);
}
/*******************************************************************************
**
*******************************************************************************/
public void addApp(String name, QAppMetaData app)
{
if(this.apps.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second app with name: " + name));
}
this.apps.put(name, app);
}
/*******************************************************************************
**
*******************************************************************************/
public QAppMetaData getApp(String name)
{
return (this.apps.get(name));
}
/*******************************************************************************
** Getter for backends
**
*******************************************************************************/
public Map<String, QBackendMetaData> getBackends()
{
return backends;
}
/*******************************************************************************
** Setter for backends
**
*******************************************************************************/
public void setBackends(Map<String, QBackendMetaData> backends)
{
this.backends = backends;
}
/*******************************************************************************
** Getter for tables
**
*******************************************************************************/
public Map<String, QTableMetaData> getTables()
{
return tables;
}
/*******************************************************************************
** Setter for tables
**
*******************************************************************************/
public void setTables(Map<String, QTableMetaData> tables)
{
this.tables = tables;
}
/*******************************************************************************
** Getter for possibleValueSources
**
*******************************************************************************/
public Map<String, QPossibleValueSource> getPossibleValueSources()
{
return possibleValueSources;
}
/*******************************************************************************
** Setter for possibleValueSources
**
*******************************************************************************/
public void setPossibleValueSources(Map<String, QPossibleValueSource> possibleValueSources)
{
this.possibleValueSources = possibleValueSources;
}
/*******************************************************************************
** Getter for processes
**
@ -393,6 +372,44 @@ public class QInstance
/*******************************************************************************
**
*******************************************************************************/
public void addApp(QAppMetaData app)
{
this.addApp(app.getName(), app);
}
/*******************************************************************************
**
*******************************************************************************/
public void addApp(String name, QAppMetaData app)
{
if(!StringUtils.hasContent(name))
{
throw (new IllegalArgumentException("Attempted to add an app without a name."));
}
if(this.apps.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second app with name: " + name));
}
this.apps.put(name, app);
}
/*******************************************************************************
**
*******************************************************************************/
public QAppMetaData getApp(String name)
{
return (this.apps.get(name));
}
/*******************************************************************************
** Getter for apps
**
@ -415,6 +432,62 @@ public class QInstance
/*******************************************************************************
**
*******************************************************************************/
public void addAutomationProvider(QAutomationProviderMetaData automationProvider)
{
this.addAutomationProvider(automationProvider.getName(), automationProvider);
}
/*******************************************************************************
**
*******************************************************************************/
public void addAutomationProvider(String name, QAutomationProviderMetaData automationProvider)
{
if(this.automationProviders.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second automationProvider with name: " + name));
}
this.automationProviders.put(name, automationProvider);
}
/*******************************************************************************
**
*******************************************************************************/
public QAutomationProviderMetaData getAutomationProvider(String name)
{
return (this.automationProviders.get(name));
}
/*******************************************************************************
** Getter for automationProviders
**
*******************************************************************************/
public Map<String, QAutomationProviderMetaData> getAutomationProviders()
{
return automationProviders;
}
/*******************************************************************************
** Setter for automationProviders
**
*******************************************************************************/
public void setAutomationProviders(Map<String, QAutomationProviderMetaData> automationProviders)
{
this.automationProviders = automationProviders;
}
/*******************************************************************************
** Getter for hasBeenValidated
**
@ -426,6 +499,39 @@ public class QInstance
/*******************************************************************************
** Setter for hasBeenValidated
**
*******************************************************************************/
public void setHasBeenValidated(QInstanceValidationKey key)
{
this.hasBeenValidated = true;
}
/*******************************************************************************
** Getter for branding
**
*******************************************************************************/
public QBrandingMetaData getBranding()
{
return branding;
}
/*******************************************************************************
** Setter for branding
**
*******************************************************************************/
public void setBranding(QBrandingMetaData branding)
{
this.branding = branding;
}
/*******************************************************************************
** Getter for authentication
**
@ -445,4 +551,61 @@ public class QInstance
{
this.authentication = authentication;
}
/*******************************************************************************
** Getter for widgets
**
*******************************************************************************/
public Map<String, QWidgetMetaDataInterface> getWidgets()
{
return widgets;
}
/*******************************************************************************
** Setter for widgets
**
*******************************************************************************/
public void setWidgets(Map<String, QWidgetMetaDataInterface> widgets)
{
this.widgets = widgets;
}
/*******************************************************************************
**
*******************************************************************************/
public void addWidget(QWidgetMetaDataInterface widget)
{
this.addWidget(widget.getName(), widget);
}
/*******************************************************************************
**
*******************************************************************************/
public void addWidget(String name, QWidgetMetaDataInterface widget)
{
if(this.widgets.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second widget with name: " + name));
}
this.widgets.put(name, widget);
}
/*******************************************************************************
**
*******************************************************************************/
public QWidgetMetaDataInterface getWidget(String name)
{
return (this.widgets.get(name));
}
}

64
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java Normal file
View File

@ -0,0 +1,64 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.automation;
/*******************************************************************************
** Metadata specifically for the polling automation provider.
*******************************************************************************/
public class PollingAutomationProviderMetaData extends QAutomationProviderMetaData
{
/*******************************************************************************
**
*******************************************************************************/
public PollingAutomationProviderMetaData()
{
super();
setType(QAutomationProviderType.POLLING);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public PollingAutomationProviderMetaData withName(String name)
{
setName(name);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public PollingAutomationProviderMetaData withType(QAutomationProviderType type)
{
setType(type);
return (this);
}
}

98
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java Normal file
View File

@ -0,0 +1,98 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.automation;
/*******************************************************************************
** Meta-data definition of a qqq service to drive record automations.
*******************************************************************************/
public class QAutomationProviderMetaData
{
private String name;
private QAutomationProviderType type;
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public QAutomationProviderMetaData withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for type
**
*******************************************************************************/
public QAutomationProviderType getType()
{
return type;
}
/*******************************************************************************
** Setter for type
**
*******************************************************************************/
public void setType(QAutomationProviderType type)
{
this.type = type;
}
/*******************************************************************************
** Fluent setter for type
**
*******************************************************************************/
public QAutomationProviderMetaData withType(QAutomationProviderType type)
{
this.type = type;
return (this);
}
}

60
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderType.java Normal file
View File

@ -0,0 +1,60 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.automation;
/*******************************************************************************
** Enum to define the possible automation provider types
**
*******************************************************************************/
public enum QAutomationProviderType
{
POLLING("polling"),
SYNCHRONOUS("synchronous"),
ASYNCHRONOUS("asynchronous"),
MQ("mq"),
AMAZON_SQS("sqs");
private final String name;
/*******************************************************************************
** enum constructor
*******************************************************************************/
QAutomationProviderType(String name)
{
this.name = name;
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return this.name;
}
}

148
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java Normal file
View File

@ -0,0 +1,148 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.branding;
/*******************************************************************************
** Meta-Data to define branding in a QQQ instance.
**
*******************************************************************************/
public class QBrandingMetaData
{
private String companyName;
private String logo;
private String icon;
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return ("QBrandingMetaData[" + companyName + "]");
}
/*******************************************************************************
** Getter for companyName
**
*******************************************************************************/
public String getCompanyName()
{
return companyName;
}
/*******************************************************************************
** Setter for companyName
**
*******************************************************************************/
public void setCompanyName(String companyName)
{
this.companyName = companyName;
}
/*******************************************************************************
** Fluent setter for companyName
**
*******************************************************************************/
public QBrandingMetaData withCompanyName(String companyName)
{
this.companyName = companyName;
return this;
}
/*******************************************************************************
** Getter for logo
**
*******************************************************************************/
public String getLogo()
{
return logo;
}
/*******************************************************************************
** Setter for logo
**
*******************************************************************************/
public void setLogo(String logo)
{
this.logo = logo;
}
/*******************************************************************************
** Fluent setter for logo
**
*******************************************************************************/
public QBrandingMetaData withLogo(String logo)
{
this.logo = logo;
return this;
}
/*******************************************************************************
** Getter for icon
**
*******************************************************************************/
public String getIcon()
{
return icon;
}
/*******************************************************************************
** Setter for icon
**
*******************************************************************************/
public void setIcon(String icon)
{
this.icon = icon;
}
/*******************************************************************************
** Fluent setter for icon
**
*******************************************************************************/
public QBrandingMetaData withIcon(String icon)
{
this.icon = icon;
return this;
}
}

19
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java
View File

@ -22,6 +22,8 @@
package com.kingsrook.qqq.backend.core.model.metadata.code;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
@ -30,7 +32,7 @@ import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvide
** Pointer to code to be ran by the qqq framework, e.g., for custom behavior -
** maybe process steps, maybe customization to a table, etc.
*******************************************************************************/
public class QCodeReference
public class QCodeReference implements Serializable
{
private String name;
private QCodeType codeType;
@ -59,6 +61,17 @@ public class QCodeReference
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return "QCodeReference{name='" + name + "'}";
}
/*******************************************************************************
** Constructor that just takes a java class, and infers the other fields.
*******************************************************************************/
@ -75,6 +88,10 @@ public class QCodeReference
{
this.codeUsage = QCodeUsage.POSSIBLE_VALUE_PROVIDER;
}
else if(RecordAutomationHandler.class.isAssignableFrom(javaClass))
{
this.codeUsage = QCodeUsage.RECORD_AUTOMATION_HANDLER;
}
else
{
throw (new IllegalStateException("Unable to infer code usage type for class: " + javaClass.getName()));

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeUsage.java
View File

@ -30,5 +30,6 @@ public enum QCodeUsage
{
BACKEND_STEP, // a backend-step in a process
CUSTOMIZER, // a function to customize part of a QQQ table's behavior
POSSIBLE_VALUE_PROVIDER // code that drives a custom possibleValueSource
POSSIBLE_VALUE_PROVIDER, // code that drives a custom possibleValueSource
RECORD_AUTOMATION_HANDLER // code that executes record automations
}

105
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java Normal file
View File

@ -0,0 +1,105 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
/*******************************************************************************
** Base metadata for frontend dashboard widgets
**
*******************************************************************************/
public class QWidgetMetaData implements QWidgetMetaDataInterface
{
protected String name;
protected QCodeReference codeReference;
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public QWidgetMetaData withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for codeReference
**
*******************************************************************************/
public QCodeReference getCodeReference()
{
return codeReference;
}
/*******************************************************************************
** Setter for codeReference
**
*******************************************************************************/
public void setCodeReference(QCodeReference codeReference)
{
this.codeReference = codeReference;
}
/*******************************************************************************
** Fluent setter for codeReference
**
*******************************************************************************/
public QWidgetMetaData withCodeReference(QCodeReference codeReference)
{
this.codeReference = codeReference;
return (this);
}
}

64
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java Normal file
View File

@ -0,0 +1,64 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
/*******************************************************************************
** Interface for qqq widget meta data
**
*******************************************************************************/
public interface QWidgetMetaDataInterface
{
/*******************************************************************************
** Getter for name
*******************************************************************************/
String getName();
/*******************************************************************************
** Setter for name
*******************************************************************************/
void setName(String name);
/*******************************************************************************
** Fluent setter for name
*******************************************************************************/
QWidgetMetaDataInterface withName(String name);
/*******************************************************************************
** Getter for codeReference
*******************************************************************************/
QCodeReference getCodeReference();
/*******************************************************************************
** Setter for codeReference
*******************************************************************************/
void setCodeReference(QCodeReference codeReference);
/*******************************************************************************
** Fluent setter for codeReference
*******************************************************************************/
QWidgetMetaDataInterface withCodeReference(QCodeReference codeReference);
}

327
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java Normal file
View File

@ -0,0 +1,327 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
import java.util.Collection;
/*******************************************************************************
** AWS Quicksite specific meta data for frontend dashboard widget
**
*******************************************************************************/
public class QuickSightChartMetaData extends QWidgetMetaData implements QWidgetMetaDataInterface
{
private String label;
private String accessKey;
private String secretKey;
private String dashboardId;
private String accountId;
private String userArn;
private String region;
private Collection<String> allowedDomains;
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public QuickSightChartMetaData withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for accessKey
**
*******************************************************************************/
public String getAccessKey()
{
return accessKey;
}
/*******************************************************************************
** Setter for accessKey
**
*******************************************************************************/
public void setAccessKey(String accessKey)
{
this.accessKey = accessKey;
}
/*******************************************************************************
** Fluent setter for accessKey
**
*******************************************************************************/
public QuickSightChartMetaData withAccessKey(String accessKey)
{
this.accessKey = accessKey;
return (this);
}
/*******************************************************************************
** Getter for label
**
*******************************************************************************/
public String getLabel()
{
return label;
}
/*******************************************************************************
** Setter for label
**
*******************************************************************************/
public void setLabel(String label)
{
this.label = label;
}
/*******************************************************************************
** Fluent setter for label
**
*******************************************************************************/
public QuickSightChartMetaData withLabel(String label)
{
this.label = label;
return (this);
}
/*******************************************************************************
** Getter for secretKey
**
*******************************************************************************/
public String getSecretKey()
{
return secretKey;
}
/*******************************************************************************
** Setter for secretKey
**
*******************************************************************************/
public void setSecretKey(String secretKey)
{
this.secretKey = secretKey;
}
/*******************************************************************************
** Fluent setter for secretKey
**
*******************************************************************************/
public QuickSightChartMetaData withSecretKey(String secretKey)
{
this.secretKey = secretKey;
return (this);
}
/*******************************************************************************
** Getter for dashboardId
**
*******************************************************************************/
public String getDashboardId()
{
return dashboardId;
}
/*******************************************************************************
** Setter for dashboardId
**
*******************************************************************************/
public void setDashboardId(String dashboardId)
{
this.dashboardId = dashboardId;
}
/*******************************************************************************
** Fluent setter for dashboardId
**
*******************************************************************************/
public QuickSightChartMetaData withDashboardId(String dashboardId)
{
this.dashboardId = dashboardId;
return (this);
}
/*******************************************************************************
** Getter for accountId
**
*******************************************************************************/
public String getAccountId()
{
return accountId;
}
/*******************************************************************************
** Setter for accountId
**
*******************************************************************************/
public void setAccountId(String accountId)
{
this.accountId = accountId;
}
/*******************************************************************************
** Fluent setter for accountId
**
*******************************************************************************/
public QuickSightChartMetaData withAccountId(String accountId)
{
this.accountId = accountId;
return this;
}
/*******************************************************************************
** Getter for userArn
**
*******************************************************************************/
public String getUserArn()
{
return userArn;
}
/*******************************************************************************
** Setter for userArn
**
*******************************************************************************/
public void setUserArn(String userArn)
{
this.userArn = userArn;
}
/*******************************************************************************
** Fluent setter for userArn
**
*******************************************************************************/
public QuickSightChartMetaData withUserArn(String userArn)
{
this.userArn = userArn;
return this;
}
/*******************************************************************************
** Getter for region
**
*******************************************************************************/
public String getRegion()
{
return region;
}
/*******************************************************************************
** Setter for region
**
*******************************************************************************/
public void setRegion(String region)
{
this.region = region;
}
/*******************************************************************************
** Fluent setter for region
**
*******************************************************************************/
public QuickSightChartMetaData withRegion(String region)
{
this.region = region;
return this;
}
/*******************************************************************************
** Getter for allowedDomains
**
*******************************************************************************/
public Collection<String> getAllowedDomains()
{
return allowedDomains;
}
/*******************************************************************************
** Setter for allowedDomains
**
*******************************************************************************/
public void setAllowedDomains(Collection<String> allowedDomains)
{
this.allowedDomains = allowedDomains;
}
/*******************************************************************************
** Fluent setter for allowedDomains
**
*******************************************************************************/
public QuickSightChartMetaData withAllowedDomains(Collection<String> allowedDomains)
{
this.allowedDomains = allowedDomains;
return this;
}
}

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java
View File

@ -76,6 +76,17 @@ public class QFieldMetaData
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return ("QFieldMetaData[" + name + "]");
}
/*******************************************************************************
** Initialize a fieldMetaData from a reference to a getter on an entity.
** e.g., new QFieldMetaData(Order::getOrderNo).

30
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java
View File

@ -26,7 +26,8 @@ import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
@ -40,6 +41,7 @@ public class QFrontendAppMetaData
private String label;
private List<AppTreeNode> children = new ArrayList<>();
private List<String> widgets = new ArrayList<>();
private String iconName;
@ -48,14 +50,19 @@ public class QFrontendAppMetaData
/*******************************************************************************
**
*******************************************************************************/
public QFrontendAppMetaData(QAppChildMetaData appChildMetaData)
public QFrontendAppMetaData(QAppMetaData appMetaData)
{
this.name = appChildMetaData.getName();
this.label = appChildMetaData.getLabel();
this.name = appMetaData.getName();
this.label = appMetaData.getLabel();
if(appChildMetaData.getIcon() != null)
if(appMetaData.getIcon() != null)
{
this.iconName = appChildMetaData.getIcon().getName();
this.iconName = appMetaData.getIcon().getName();
}
if(CollectionUtils.nullSafeHasContents(appMetaData.getWidgets()))
{
this.widgets = appMetaData.getWidgets();
}
}
@ -127,4 +134,15 @@ public class QFrontendAppMetaData
}
children.add(childAppTreeNode);
}
/*******************************************************************************
** Getter for widgets
**
*******************************************************************************/
public List<String> getWidgets()
{
return widgets;
}
}

35
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java
View File

@ -40,6 +40,7 @@ public class QAppMetaData implements QAppChildMetaData
private String parentAppName;
private QIcon icon;
private List<String> widgets;
/*******************************************************************************
@ -235,4 +236,38 @@ public class QAppMetaData implements QAppChildMetaData
return (this);
}
/*******************************************************************************
** Getter for widgets
**
*******************************************************************************/
public List<String> getWidgets()
{
return widgets;
}
/*******************************************************************************
** Setter for widgets
**
*******************************************************************************/
public void setWidgets(List<String> widgets)
{
this.widgets = widgets;
}
/*******************************************************************************
** Fluent setter for widgets
**
*******************************************************************************/
public QAppMetaData withWidgets(List<String> widgets)
{
this.widgets = widgets;
return (this);
}
}

21
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PVSValueFormatAndFields.java
View File

@ -1,3 +1,24 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues;

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java
View File

@ -28,7 +28,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes;
public enum QComponentType
{
HELP_TEXT,
BULK_EDIT_FORM;
BULK_EDIT_FORM,
VALIDATION_REVIEW_SCREEN,
EDIT_FORM,
VIEW_FORM,
RECORD_LIST,
PROCESS_SUMMARY_RESULTS;
///////////////////////////////////////////////////////////////////////////
// keep these values in sync with QComponentType.ts in qqq-frontend-core //
///////////////////////////////////////////////////////////////////////////

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionOutputMetaData.java
View File

@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
*******************************************************************************/
public class QFunctionOutputMetaData
{
private QRecordListMetaData recordListMetaData;
private QRecordListMetaData recordListMetaData;
private List<QFieldMetaData> fieldList;
@ -106,11 +106,12 @@ public class QFunctionOutputMetaData
/*******************************************************************************
** Setter for fieldList
** Fluently add a field to the list
**
*******************************************************************************/
public QFunctionOutputMetaData addField(QFieldMetaData field)
public QFunctionOutputMetaData withField(QFieldMetaData field)
{
if(this.fieldList == null)
{

126
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java
View File

@ -23,7 +23,11 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
@ -41,13 +45,25 @@ public class QProcessMetaData implements QAppChildMetaData
private String tableName;
private boolean isHidden = false;
private List<QStepMetaData> stepList;
private List<QStepMetaData> stepList; // these are the steps that are ran, by-default, in the order they are ran in
private Map<String, QStepMetaData> steps; // this is the full map of possible steps
private String parentAppName;
private QIcon icon;
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return ("QProcessMetaData[" + name + "]");
}
/*******************************************************************************
** Getter for name
**
@ -167,23 +183,69 @@ public class QProcessMetaData implements QAppChildMetaData
*******************************************************************************/
public QProcessMetaData withStepList(List<QStepMetaData> stepList)
{
this.stepList = stepList;
if(stepList != null)
{
stepList.forEach(this::addStep);
}
return (this);
}
/*******************************************************************************
** Setter for stepList
** add a step to the stepList and map
**
*******************************************************************************/
public QProcessMetaData addStep(QStepMetaData step)
{
int index = 0;
if(this.stepList != null)
{
index = this.stepList.size();
}
addStep(index, step);
return (this);
}
/*******************************************************************************
** add a step to the stepList (at the specified index) and the step map
**
*******************************************************************************/
public QProcessMetaData addStep(int index, QStepMetaData step)
{
if(this.stepList == null)
{
this.stepList = new ArrayList<>();
}
this.stepList.add(step);
this.stepList.add(index, step);
if(this.steps == null)
{
this.steps = new HashMap<>();
}
this.steps.put(step.getName(), step);
return (this);
}
/*******************************************************************************
** add a step ONLY to the step map - NOT the list w/ default execution order.
**
*******************************************************************************/
public QProcessMetaData addOptionalStep(QStepMetaData step)
{
if(this.steps == null)
{
this.steps = new HashMap<>();
}
this.steps.put(step.getName(), step);
return (this);
}
@ -205,15 +267,7 @@ public class QProcessMetaData implements QAppChildMetaData
*******************************************************************************/
public QStepMetaData getStep(String stepName)
{
for(QStepMetaData step : stepList)
{
if(step.getName().equals(stepName))
{
return (step);
}
}
return (null);
return (steps.get(stepName));
}
@ -229,17 +283,35 @@ public class QProcessMetaData implements QAppChildMetaData
/*******************************************************************************
** Get a list of all of the input fields used by all the steps in this process.
** Wrapper to getStep, that internally casts to FrontendStepMetaData
*******************************************************************************/
public QFrontendStepMetaData getFrontendStep(String name)
{
return (QFrontendStepMetaData) getStep(name);
}
/*******************************************************************************
** Get a list of all the *unique* input fields used by all the steps in this process.
*******************************************************************************/
@JsonIgnore
public List<QFieldMetaData> getInputFields()
{
List<QFieldMetaData> rs = new ArrayList<>();
if(stepList != null)
Set<String> usedFieldNames = new HashSet<>();
List<QFieldMetaData> rs = new ArrayList<>();
if(steps != null)
{
for(QStepMetaData step : stepList)
for(QStepMetaData step : steps.values())
{
rs.addAll(step.getInputFields());
for(QFieldMetaData field : step.getInputFields())
{
if(!usedFieldNames.contains(field.getName()))
{
rs.add(field);
usedFieldNames.add(field.getName());
}
}
}
}
return (rs);
@ -248,17 +320,25 @@ public class QProcessMetaData implements QAppChildMetaData
/*******************************************************************************
** Get a list of all of the output fields used by all the steps in this process.
** Get a list of all the *unique* output fields used by all the steps in this process.
*******************************************************************************/
@JsonIgnore
public List<QFieldMetaData> getOutputFields()
{
List<QFieldMetaData> rs = new ArrayList<>();
if(stepList != null)
Set<String> usedFieldNames = new HashSet<>();
List<QFieldMetaData> rs = new ArrayList<>();
if(steps != null)
{
for(QStepMetaData step : stepList)
for(QStepMetaData step : steps.values())
{
rs.addAll(step.getOutputFields());
for(QFieldMetaData field : step.getOutputFields())
{
if(!usedFieldNames.contains(field.getName()))
{
rs.add(field);
usedFieldNames.add(field.getName());
}
}
}
}
return (rs);

38
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java
View File

@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
/*******************************************************************************
@ -63,7 +64,8 @@ public class QTableMetaData implements QAppChildMetaData, Serializable
private Map<String, QFieldMetaData> fields;
private QTableBackendDetails backendDetails;
private QTableBackendDetails backendDetails;
private QTableAutomationDetails automationDetails;
private Map<String, QCodeReference> customizers;
@ -410,6 +412,40 @@ public class QTableMetaData implements QAppChildMetaData, Serializable
/*******************************************************************************
** Getter for automationDetails
**
*******************************************************************************/
public QTableAutomationDetails getAutomationDetails()
{
return automationDetails;
}
/*******************************************************************************
** Setter for automationDetails
**
*******************************************************************************/
public void setAutomationDetails(QTableAutomationDetails automationDetails)
{
this.automationDetails = automationDetails;
}
/*******************************************************************************
** Fluent Setter for automationDetails
**
*******************************************************************************/
public QTableMetaData withAutomationDetails(QTableAutomationDetails automationDetails)
{
this.automationDetails = automationDetails;
return (this);
}
/*******************************************************************************
**
*******************************************************************************/

105
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java Normal file
View File

@ -0,0 +1,105 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
/*******************************************************************************
** Table-automation meta-data to define how this table's per-record automation
** status is tracked.
*******************************************************************************/
public class AutomationStatusTracking
{
private AutomationStatusTrackingType type;
private String fieldName; // used when type is FIELD_IN_TABLE
// todo - fields for additional types (e.g., 1-1 table, shared-table)
/*******************************************************************************
** Getter for type
**
*******************************************************************************/
public AutomationStatusTrackingType getType()
{
return type;
}
/*******************************************************************************
** Setter for type
**
*******************************************************************************/
public void setType(AutomationStatusTrackingType type)
{
this.type = type;
}
/*******************************************************************************
** Fluent setter for type
**
*******************************************************************************/
public AutomationStatusTracking withType(AutomationStatusTrackingType type)
{
this.type = type;
return (this);
}
/*******************************************************************************
** Getter for fieldName
**
*******************************************************************************/
public String getFieldName()
{
return fieldName;
}
/*******************************************************************************
** Setter for fieldName
**
*******************************************************************************/
public void setFieldName(String fieldName)
{
this.fieldName = fieldName;
}
/*******************************************************************************
** Fluent setter for fieldName
**
*******************************************************************************/
public AutomationStatusTracking withFieldName(String fieldName)
{
this.fieldName = fieldName;
return (this);
}
}

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java Normal file
View File

@ -0,0 +1,32 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
/*******************************************************************************
** Enum of possible types of per-table, per-record automation-status tracking.
*******************************************************************************/
public enum AutomationStatusTrackingType
{
FIELD_IN_TABLE
// todo - additional types (e.g., 1-1 table, shared-table)
}

154
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java Normal file
View File

@ -0,0 +1,154 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
import java.util.ArrayList;
import java.util.List;
/*******************************************************************************
** Details about how this table's record automations are set up.
*******************************************************************************/
public class QTableAutomationDetails
{
private AutomationStatusTracking statusTracking;
private String providerName;
private List<TableAutomationAction> actions;
/*******************************************************************************
** Getter for statusTracking
**
*******************************************************************************/
public AutomationStatusTracking getStatusTracking()
{
return statusTracking;
}
/*******************************************************************************
** Setter for statusTracking
**
*******************************************************************************/
public void setStatusTracking(AutomationStatusTracking statusTracking)
{
this.statusTracking = statusTracking;
}
/*******************************************************************************
** Fluent setter for statusTracking
**
*******************************************************************************/
public QTableAutomationDetails withStatusTracking(AutomationStatusTracking statusTracking)
{
this.statusTracking = statusTracking;
return (this);
}
/*******************************************************************************
** Getter for providerName
**
*******************************************************************************/
public String getProviderName()
{
return providerName;
}
/*******************************************************************************
** Setter for providerName
**
*******************************************************************************/
public void setProviderName(String providerName)
{
this.providerName = providerName;
}
/*******************************************************************************
** Fluent setter for providerName
**
*******************************************************************************/
public QTableAutomationDetails withProviderName(String providerName)
{
this.providerName = providerName;
return (this);
}
/*******************************************************************************
** Getter for actions
**
*******************************************************************************/
public List<TableAutomationAction> getActions()
{
return actions;
}
/*******************************************************************************
** Setter for actions
**
*******************************************************************************/
public void setActions(List<TableAutomationAction> actions)
{
this.actions = actions;
}
/*******************************************************************************
** Fluent setter for actions
**
*******************************************************************************/
public QTableAutomationDetails withActions(List<TableAutomationAction> actions)
{
this.actions = actions;
return (this);
}
/*******************************************************************************
** Fluently add an action to this table's automations.
*******************************************************************************/
public QTableAutomationDetails withAction(TableAutomationAction action)
{
if(this.actions == null)
{
this.actions = new ArrayList<>();
}
this.actions.add(action);
return (this);
}
}

253
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java Normal file
View File

@ -0,0 +1,253 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
/*******************************************************************************
** Definition of a specific action to run against a table
*******************************************************************************/
public class TableAutomationAction
{
private String name;
private TriggerEvent triggerEvent;
private Integer priority = 500;
private QQueryFilter filter;
////////////////////////////////
// mutually-exclusive options //
////////////////////////////////
private QCodeReference codeReference;
private String processName;
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return "TableAutomationAction{name='" + name + "'}";}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public TableAutomationAction withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for triggerEvent
**
*******************************************************************************/
public TriggerEvent getTriggerEvent()
{
return triggerEvent;
}
/*******************************************************************************
** Setter for triggerEvent
**
*******************************************************************************/
public void setTriggerEvent(TriggerEvent triggerEvent)
{
this.triggerEvent = triggerEvent;
}
/*******************************************************************************
** Fluent setter for triggerEvent
**
*******************************************************************************/
public TableAutomationAction withTriggerEvent(TriggerEvent triggerEvent)
{
this.triggerEvent = triggerEvent;
return (this);
}
/*******************************************************************************
** Getter for priority
**
*******************************************************************************/
public Integer getPriority()
{
return priority;
}
/*******************************************************************************
** Setter for priority
**
*******************************************************************************/
public void setPriority(Integer priority)
{
this.priority = priority;
}
/*******************************************************************************
** Fluent setter for priority
**
*******************************************************************************/
public TableAutomationAction withPriority(Integer priority)
{
this.priority = priority;
return (this);
}
/*******************************************************************************
** Getter for filter
**
*******************************************************************************/
public QQueryFilter getFilter()
{
return filter;
}
/*******************************************************************************
** Setter for filter
**
*******************************************************************************/
public void setFilter(QQueryFilter filter)
{
this.filter = filter;
}
/*******************************************************************************
** Fluent setter for filter
**
*******************************************************************************/
public TableAutomationAction withFilter(QQueryFilter filter)
{
this.filter = filter;
return (this);
}
/*******************************************************************************
** Getter for codeReference
**
*******************************************************************************/
public QCodeReference getCodeReference()
{
return codeReference;
}
/*******************************************************************************
** Setter for codeReference
**
*******************************************************************************/
public void setCodeReference(QCodeReference codeReference)
{
this.codeReference = codeReference;
}
/*******************************************************************************
** Fluent setter for codeReference
**
*******************************************************************************/
public TableAutomationAction withCodeReference(QCodeReference codeReference)
{
this.codeReference = codeReference;
return (this);
}
/*******************************************************************************
** Getter for processName
**
*******************************************************************************/
public String getProcessName()
{
return processName;
}
/*******************************************************************************
** Setter for processName
**
*******************************************************************************/
public void setProcessName(String processName)
{
this.processName = processName;
}
/*******************************************************************************
** Fluent setter for processName
**
*******************************************************************************/
public TableAutomationAction withProcessName(String processName)
{
this.processName = processName;
return (this);
}
}

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java Normal file
View File

@ -0,0 +1,33 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
/*******************************************************************************
** Possible events that can trigger a record automation.
*******************************************************************************/
public enum TriggerEvent
{
POST_INSERT,
POST_UPDATE,
PRE_DELETE
}

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java
View File

@ -35,6 +35,10 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
** A simple (probably only valid for testing?) implementation of the QModuleInterface,
** that just stores its records in-memory.
**
** In general, this class is intended to behave, as much as possible, like an RDBMS.
**
** TODO - in future, if we need to - make configs for things like "case-insensitive",
** and "allow loose typing".
*******************************************************************************/
public class MemoryBackendModule implements QBackendModuleInterface
{

251
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import java.io.Serializable;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -32,12 +33,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.commons.lang.NotImplementedException;
@ -121,42 +124,7 @@ public class MemoryRecordStore
for(QRecord qRecord : tableData.values())
{
boolean recordMatches = true;
if(input.getFilter() != null && input.getFilter().getCriteria() != null)
{
for(QFilterCriteria criterion : input.getFilter().getCriteria())
{
String fieldName = criterion.getFieldName();
Serializable value = qRecord.getValue(fieldName);
switch(criterion.getOperator())
{
case EQUALS:
{
if(!value.equals(criterion.getValues().get(0)))
{
recordMatches = false;
}
break;
}
case IN:
{
if(!criterion.getValues().contains(value))
{
recordMatches = false;
}
break;
}
default:
{
throw new NotImplementedException("Operator [" + criterion.getOperator() + "] is not yet implemented in the Memory backend.");
}
}
if(!recordMatches)
{
break;
}
}
}
boolean recordMatches = doesRecordMatch(input.getFilter(), qRecord);
if(recordMatches)
{
@ -169,6 +137,217 @@ public class MemoryRecordStore
/*******************************************************************************
**
*******************************************************************************/
private boolean doesRecordMatch(QQueryFilter filter, QRecord qRecord)
{
boolean recordMatches = true;
if(filter != null && filter.getCriteria() != null)
{
for(QFilterCriteria criterion : filter.getCriteria())
{
String fieldName = criterion.getFieldName();
Serializable value = qRecord.getValue(fieldName);
switch(criterion.getOperator())
{
case EQUALS:
{
recordMatches = testEquals(criterion, value);
break;
}
case NOT_EQUALS:
{
recordMatches = !testEquals(criterion, value);
break;
}
case IN:
{
recordMatches = testIn(criterion, value);
break;
}
case NOT_IN:
{
recordMatches = !testIn(criterion, value);
break;
}
case CONTAINS:
{
recordMatches = testContains(criterion, fieldName, value);
break;
}
case NOT_CONTAINS:
{
recordMatches = !testContains(criterion, fieldName, value);
break;
}
case GREATER_THAN:
{
recordMatches = testGreaterThan(criterion, value);
break;
}
case GREATER_THAN_OR_EQUALS:
{
recordMatches = testGreaterThan(criterion, value) || testEquals(criterion, value);
break;
}
case LESS_THAN:
{
recordMatches = !testGreaterThan(criterion, value) && !testEquals(criterion, value);
break;
}
case LESS_THAN_OR_EQUALS:
{
recordMatches = !testGreaterThan(criterion, value);
break;
}
default:
{
throw new NotImplementedException("Operator [" + criterion.getOperator() + "] is not yet implemented in the Memory backend.");
}
}
if(!recordMatches)
{
break;
}
}
}
return recordMatches;
}
/*******************************************************************************
**
*******************************************************************************/
private boolean testGreaterThan(QFilterCriteria criterion, Serializable value)
{
Serializable criterionValue = criterion.getValues().get(0);
if(criterionValue == null)
{
throw (new IllegalArgumentException("Missing criterion value in query"));
}
if (value == null)
{
/////////////////////////////////////////////////////////////////////////////////////
// a database would say 'false' for if a null column is > a value, so do the same. //
/////////////////////////////////////////////////////////////////////////////////////
return (false);
}
if(value instanceof LocalDate valueDate && criterionValue instanceof LocalDate criterionValueDate)
{
return (valueDate.isAfter(criterionValueDate));
}
if(value instanceof Number valueNumber && criterionValue instanceof Number criterionValueNumber)
{
return (valueNumber.doubleValue() > criterionValueNumber.doubleValue());
}
throw (new NotImplementedException("Greater/Less Than comparisons are not (yet?) implemented for the supplied types [" + value.getClass().getSimpleName() + "][" + criterionValue.getClass().getSimpleName() + "]"));
}
/*******************************************************************************
**
*******************************************************************************/
private boolean testIn(QFilterCriteria criterion, Serializable value)
{
if(!criterion.getValues().contains(value))
{
return (false);
}
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
private boolean testEquals(QFilterCriteria criterion, Serializable value)
{
if(value == null)
{
return (false);
}
if(!value.equals(criterion.getValues().get(0)))
{
return (false);
}
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
private boolean testContains(QFilterCriteria criterion, String fieldName, Serializable value)
{
String stringValue = getStringFieldValue(value, fieldName, criterion);
String criterionValue = getFirstStringCriterionValue(criterion);
if(!stringValue.contains(criterionValue))
{
return (false);
}
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
private String getFirstStringCriterionValue(QFilterCriteria criteria)
{
if(CollectionUtils.nullSafeIsEmpty(criteria.getValues()))
{
throw new IllegalArgumentException("Missing value for [" + criteria.getOperator() + "] criteria on field [" + criteria.getFieldName() + "]");
}
Serializable value = criteria.getValues().get(0);
if(value == null)
{
return "";
}
if(!(value instanceof String stringValue))
{
throw new ClassCastException("Value [" + value + "] for criteria [" + criteria.getFieldName() + "] is not a String, which is required for the [" + criteria.getOperator() + "] operator.");
}
return (stringValue);
}
/*******************************************************************************
**
*******************************************************************************/
private String getStringFieldValue(Serializable value, String fieldName, QFilterCriteria criterion)
{
if(value == null)
{
return "";
}
if(!(value instanceof String stringValue))
{
throw new ClassCastException("Value [" + value + "] in field [" + fieldName + "] is not a String, which is required for the [" + criterion.getOperator() + "] operator.");
}
return (stringValue);
}
/*******************************************************************************
**
*******************************************************************************/

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java
View File

@ -35,6 +35,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -43,6 +45,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
*******************************************************************************/
public class MockQueryAction implements QueryInterface
{
private static final Logger LOG = LogManager.getLogger(MockQueryAction.class);
/*******************************************************************************
**
@ -68,6 +73,12 @@ public class MockQueryAction implements QueryInterface
}
queryOutput.addRecord(record);
if(queryInput.getAsyncJobCallback().wasCancelRequested())
{
LOG.info("Breaking query job, as requested.");
break;
}
}
return (queryOutput);

101
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java
View File

@ -1,101 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import java.io.IOException;
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Backend step to do a bulk delete.
*******************************************************************************/
public class BulkDeleteStoreStep implements BackendStep
{
public static final String ERROR_COUNT = "errorCount";
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records...");
runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal();
DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance());
deleteInput.setSession(runBackendStepInput.getSession());
deleteInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
deleteInput.setTableName(runBackendStepInput.getTableName());
String queryFilterJSON = runBackendStepInput.getValueString("queryFilterJSON");
if(StringUtils.hasContent(queryFilterJSON))
{
try
{
deleteInput.setQueryFilter(JsonUtils.toObject(queryFilterJSON, QQueryFilter.class));
}
catch(IOException e)
{
throw (new QException("Error loading record query filter from process", e));
}
}
else if(CollectionUtils.nullSafeHasContents(runBackendStepInput.getRecords()))
{
String primaryKeyField = runBackendStepInput.getTable().getPrimaryKeyField();
List<Serializable> primaryKeyList = runBackendStepInput.getRecords().stream()
.map(r -> r.getValue(primaryKeyField))
.toList();
deleteInput.setPrimaryKeys(primaryKeyList);
}
else
{
throw (new QException("Missing required inputs (queryFilterJSON or record list)"));
}
DeleteAction deleteAction = new DeleteAction();
DeleteOutput deleteOutput = deleteAction.execute(deleteInput);
List<QRecord> recordsWithErrors = Objects.requireNonNullElse(deleteOutput.getRecordsWithErrors(), Collections.emptyList());
runBackendStepOutput.addValue(ERROR_COUNT, recordsWithErrors.size());
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
}
}

126
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java Normal file
View File

@ -0,0 +1,126 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
/*******************************************************************************
** Transform step for generic table bulk-insert ETL process
*******************************************************************************/
public class BulkDeleteTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private String tableLabel;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
///////////////////////////////////////////////////////
// capture the table label - for the process summary //
///////////////////////////////////////////////////////
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
if(table != null)
{
tableLabel = table.getLabel();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// on the validate step, we haven't read the full file, so we don't know how many rows there are - thus //
// record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
{
if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record " + "%,d".formatted(okSummary.getCount()));
}
else
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record");
}
}
else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE))
{
if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " record " + "%,d".formatted(okSummary.getCount()));
}
else
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " records");
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// no transformation needs done - just pass records through from input to output, and assume all are OK //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
okSummary.incrementCount(runBackendStepInput.getRecords().size());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
{
if(isForResultScreen)
{
okSummary.setMessage(tableLabel + " records were deleted.");
}
else
{
okSummary.setMessage(tableLabel + " records will be deleted.");
}
ArrayList<ProcessSummaryLine> rs = new ArrayList<>();
rs.add(okSummary);
return (rs);
}
}

70
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java
View File

@ -1,70 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Backend step to receive values for a bulk edit.
*******************************************************************************/
public class BulkEditReceiveValuesStep implements BackendStep
{
public static final String FIELD_ENABLED_FIELDS = "bulkEditEnabledFields";
public static final String FIELD_VALUES_BEING_UPDATED = "valuesBeingUpdated";
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
String enabledFieldsString = runBackendStepInput.getValueString(FIELD_ENABLED_FIELDS);
String[] enabledFields = enabledFieldsString.split(",");
////////////////////////////////////////////////////////////////////////////////////////////
// put the value in all the records (note, this is just for display on the review screen, //
// and/or if we wanted to do some validation - this is NOT what will be store, as the //
// Update action only wants fields that are being changed. //
////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
{
for(String fieldName : enabledFields)
{
Serializable value = runBackendStepInput.getValue(fieldName);
record.setValue(fieldName, value);
}
}
BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "will be");
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
}
}

90
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java
View File

@ -1,90 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Backend step to store the records from a bulk insert file
*******************************************************************************/
public class BulkEditStoreRecordsStep implements BackendStep
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
String enabledFieldsString = runBackendStepInput.getValueString(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS);
String[] enabledFields = enabledFieldsString.split(",");
QTableMetaData table = runBackendStepInput.getTable();
List<QRecord> recordsToUpdate = new ArrayList<>();
runBackendStepInput.getAsyncJobCallback().updateStatus("Updating values in records...");
int i = 1;
for(QRecord record : runBackendStepInput.getRecords())
{
runBackendStepInput.getAsyncJobCallback().updateStatus(i++, runBackendStepInput.getRecords().size());
QRecord recordToUpdate = new QRecord();
recordsToUpdate.add(recordToUpdate);
recordToUpdate.setValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField()));
for(String fieldName : enabledFields)
{
Serializable value = runBackendStepInput.getValue(fieldName);
recordToUpdate.setValue(fieldName, value);
}
}
runBackendStepInput.getAsyncJobCallback().updateStatus("Storing updated records...");
runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal();
UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance());
updateInput.setSession(runBackendStepInput.getSession());
updateInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
updateInput.setAreAllValuesBeingUpdatedTheSame(true);
updateInput.setTableName(runBackendStepInput.getTableName());
updateInput.setRecords(recordsToUpdate);
UpdateAction updateAction = new UpdateAction();
UpdateOutput updateOutput = updateAction.execute(updateInput);
runBackendStepOutput.setRecords(updateOutput.getRecords());
BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "was");
}
}

215
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java Normal file
View File

@ -0,0 +1,215 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** Transform step for generic table bulk-edit ETL process
*******************************************************************************/
public class BulkEditTransformStep extends AbstractTransformStep
{
public static final String FIELD_ENABLED_FIELDS = "bulkEditEnabledFields";
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private List<ProcessSummaryLine> infoSummaries = new ArrayList<>();
private QTableMetaData table;
private String tableLabel;
private String[] enabledFields;
private boolean isValidateStep;
private boolean isExecuteStep;
private boolean haveRecordCount;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
///////////////////////////////////////////////////////
// capture the table label - for the process summary //
///////////////////////////////////////////////////////
table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
if(table != null)
{
tableLabel = table.getLabel();
}
String enabledFieldsString = runBackendStepInput.getValueString(FIELD_ENABLED_FIELDS);
enabledFields = enabledFieldsString.split(",");
isValidateStep = runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
isExecuteStep = runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE);
haveRecordCount = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) != null;
buildInfoSummaryLines(runBackendStepInput, enabledFields);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// on the validate step, we haven't read the full file, so we don't know how many rows there are - thus //
// record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(isValidateStep)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records");
if(!haveRecordCount)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing record " + "%,d".formatted(okSummary.getCount()));
}
}
else if(isExecuteStep)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Editing " + tableLabel + " records");
if(!haveRecordCount)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Editing " + tableLabel + " record " + "%,d".formatted(okSummary.getCount()));
}
}
List<QRecord> outputRecords = new ArrayList<>();
if(isExecuteStep)
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// for the execute step - create new record objects, just with the primary key, and the fields being updated. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
{
QRecord recordToUpdate = new QRecord();
recordToUpdate.setValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField()));
outputRecords.add(recordToUpdate);
setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, recordToUpdate);
}
}
else
{
////////////////////////////////////////////////////////////////////////////////////////////
// put the value in all the records (note, this is just for display on the review screen, //
// and/or if we wanted to do some validation - this is NOT what will be store, as the //
// Update action only wants fields that are being changed. //
////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
{
outputRecords.add(record);
setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, record);
}
}
runBackendStepOutput.setRecords(outputRecords);
okSummary.incrementCount(runBackendStepInput.getRecords().size());
}
/*******************************************************************************
**
*******************************************************************************/
private void buildInfoSummaryLines(RunBackendStepInput runBackendStepInput, String[] enabledFields)
{
QValueFormatter qValueFormatter = new QValueFormatter();
for(String fieldName : enabledFields)
{
QFieldMetaData field = table.getField(fieldName);
String label = field.getLabel();
Serializable value = runBackendStepInput.getValue(fieldName);
ProcessSummaryLine summaryLine = new ProcessSummaryLine(Status.INFO);
summaryLine.setCount(null);
infoSummaries.add(summaryLine);
String verb = isExecuteStep ? "was" : "will be";
if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
{
String formattedValue = qValueFormatter.formatValue(field, value); // todo - PVS!
summaryLine.setMessage(label + " " + verb + " set to: " + formattedValue);
}
else
{
summaryLine.setMessage(label + " " + verb + " cleared out");
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private void setUpdatedFieldsInRecord(RunBackendStepInput runBackendStepInput, String[] enabledFields, QRecord record)
{
for(String fieldName : enabledFields)
{
Serializable value = runBackendStepInput.getValue(fieldName);
record.setValue(fieldName, value);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
{
if(isForResultScreen)
{
okSummary.setMessage(tableLabel + " records were edited.");
}
else
{
okSummary.setMessage(tableLabel + " records will be edited.");
}
ArrayList<ProcessSummaryLine> rs = new ArrayList<>();
rs.add(okSummary);
rs.addAll(infoSummaries);
return (rs);
}
}

68
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java
View File

@ -1,68 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** Utility methods used for Bulk Edit steps
*******************************************************************************/
public class BulkEditUtils
{
/*******************************************************************************
**
*******************************************************************************/
public static void setFieldValuesBeingUpdated(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, String[] enabledFields, String verb)
{
/////////////////////////////////////////////////////////////////////
// build the string to show the user what fields are being changed //
/////////////////////////////////////////////////////////////////////
List<String> valuesBeingUpdated = new ArrayList<>();
QTableMetaData table = runBackendStepInput.getTable();
for(String fieldName : enabledFields)
{
String label = table.getField(fieldName).getLabel();
Serializable value = runBackendStepInput.getValue(fieldName);
if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
{
valuesBeingUpdated.add(label + " " + verb + " set to: " + value);
}
else
{
valuesBeingUpdated.add(label + " " + verb + " cleared out");
}
}
runBackendStepOutput.addValue(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated));
}
}

57
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java
View File

@ -31,23 +31,22 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
import com.kingsrook.qqq.backend.core.state.AbstractStateKey;
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
/*******************************************************************************
** Utility methods used by bulk insert steps
** Extract step for generic table bulk-insert ETL process
*******************************************************************************/
public class BulkInsertUtils
public class BulkInsertExtractStep extends AbstractExtractStep
{
/*******************************************************************************
**
*******************************************************************************/
static List<QRecord> getQRecordsFromFile(RunBackendStepInput runBackendStepInput) throws QException
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME);
Optional<QUploadedFile> optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey);
@ -70,33 +69,35 @@ public class BulkInsertUtils
mapping.addMapping(entry.getKey(), entry.getValue().getLabel());
}
List<QRecord> qRecords;
//////////////////////////////////////////////////////////////////////////
// get the non-editable fields - they'll be blanked out in a customizer //
//////////////////////////////////////////////////////////////////////////
List<QFieldMetaData> nonEditableFields = table.getFields().values().stream()
.filter(f -> !f.getIsEditable())
.toList();
if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv"))
{
qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping);
new CsvToQRecordAdapter().buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
.withRecordPipe(getRecordPipe())
.withLimit(getLimit())
.withCsv(new String(bytes))
.withTable(runBackendStepInput.getInstance().getTable(tableName))
.withMapping(mapping)
.withRecordCustomizer((record) ->
{
////////////////////////////////////////////
// remove values from non-editable fields //
////////////////////////////////////////////
for(QFieldMetaData nonEditableField : nonEditableFields)
{
record.setValue(nonEditableField.getName(), null);
}
}));
}
else
{
throw (new QUserFacingException("Unsupported file type."));
}
////////////////////////////////////////////////
// remove values from any non-editable fields //
////////////////////////////////////////////////
List<QFieldMetaData> nonEditableFields = table.getFields().values().stream()
.filter(f -> !f.getIsEditable())
.toList();
if(!nonEditableFields.isEmpty())
{
for(QRecord qRecord : qRecords)
{
for(QFieldMetaData nonEditableField : nonEditableFields)
{
qRecord.setValue(nonEditableField.getName(), null);
}
}
}
return (qRecords);
}
}

119
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java Normal file
View File

@ -0,0 +1,119 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
/*******************************************************************************
** Transform step for generic table bulk-insert ETL process
*******************************************************************************/
public class BulkInsertTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private String tableLabel;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
///////////////////////////////////////////////////////
// capture the table label - for the process summary //
///////////////////////////////////////////////////////
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
if(table != null)
{
tableLabel = table.getLabel();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// on the validate step, we haven't read the full file, so we don't know how many rows there are - thus //
// record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(okSummary.getCount()));
}
else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE))
{
if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null)
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " record " + "%,d".formatted(okSummary.getCount()));
}
else
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " records");
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// no transformation needs done - just pass records through from input to output, and assume all are OK //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
okSummary.incrementCount(runBackendStepInput.getRecords().size());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
{
if(isForResultScreen)
{
okSummary.setMessage(tableLabel + " records were inserted.");
}
else
{
okSummary.setMessage(tableLabel + " records will be inserted.");
}
ArrayList<ProcessSummaryLine> rs = new ArrayList<>();
rs.add(okSummary);
return (rs);
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java
View File

@ -83,7 +83,7 @@ public class BasicETLProcess
.withInputData(new QFunctionInputMetaData()
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)))
.withOutputMetaData(new QFunctionOutputMetaData()
.addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));
.withField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));
return new QProcessMetaData()
.withName(PROCESS_NAME)

101
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java
View File

@ -23,12 +23,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@ -41,7 +37,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicE
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLLoadFunction;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLTransformFunction;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -64,6 +59,7 @@ public class StreamedETLBackendStep implements BackendStep
**
*******************************************************************************/
@Override
@SuppressWarnings("checkstyle:indentation")
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
QBackendTransaction transaction = openTransaction(runBackendStepInput);
@ -74,96 +70,23 @@ public class StreamedETLBackendStep implements BackendStep
BasicETLExtractFunction basicETLExtractFunction = new BasicETLExtractFunction();
basicETLExtractFunction.setRecordPipe(recordPipe);
//////////////////////////////////////////
// run the query action as an async job //
//////////////////////////////////////////
AsyncJobManager asyncJobManager = new AsyncJobManager();
String queryJobUUID = asyncJobManager.startJob("StreamedETL>QueryAction", (status) ->
{
basicETLExtractFunction.run(runBackendStepInput, runBackendStepOutput);
return (runBackendStepOutput);
});
LOG.info("Started query job [" + queryJobUUID + "] for streamed ETL");
AsyncJobState queryJobState = AsyncJobState.RUNNING;
AsyncJobStatus asyncJobStatus = null;
long recordCount = 0;
int nextSleepMillis = INIT_SLEEP_MS;
long lastReceivedRecordsAt = System.currentTimeMillis();
long jobStartTime = System.currentTimeMillis();
while(queryJobState.equals(AsyncJobState.RUNNING))
{
if(recordPipe.countAvailableRecords() == 0)
////////////////////////////////////
// run the async-record-pipe loop //
////////////////////////////////////
int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Extract", null, recordPipe, (status) ->
{
///////////////////////////////////////////////////////////
// if the pipe is empty, sleep to let the producer work. //
// todo - smarter sleep? like get notified vs. sleep? //
///////////////////////////////////////////////////////////
LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work");
SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS);
basicETLExtractFunction.run(runBackendStepInput, runBackendStepOutput);
return (runBackendStepOutput);
},
() -> (consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction))
);
long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt;
if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS)
{
throw (new QException("Query action appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago)."));
}
}
else
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. //
////////////////////////////////////////////////////////////////////////////////////////////////////////
lastReceivedRecordsAt = System.currentTimeMillis();
nextSleepMillis = INIT_SLEEP_MS;
recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction);
LOG.info(String.format("Processed %,d records so far", recordCount));
}
////////////////////////////////////
// refresh the query job's status //
////////////////////////////////////
Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(queryJobUUID);
if(optionalAsyncJobStatus.isEmpty())
{
/////////////////////////////////////////////////
// todo - ... maybe some version of try-again? //
/////////////////////////////////////////////////
throw (new QException("Could not get status of report query job [" + queryJobUUID + "]"));
}
asyncJobStatus = optionalAsyncJobStatus.get();
queryJobState = asyncJobStatus.getState();
}
LOG.info("Query job [" + queryJobUUID + "] for ETL completed with status: " + asyncJobStatus);
/////////////////////////////////////////
// propagate errors from the query job //
/////////////////////////////////////////
if(asyncJobStatus.getState().equals(AsyncJobState.ERROR))
{
throw (new QException("Query job failed with an error", asyncJobStatus.getCaughtException()));
}
//////////////////////////////////////////////////////
// send the final records to transform & load steps //
//////////////////////////////////////////////////////
recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction);
runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount);
/////////////////////
// commit the work //
/////////////////////
transaction.commit();
long reportEndTime = System.currentTimeMillis();
LOG.info(String.format("Processed %,d records", recordCount)
+ String.format(" at end of ETL job in %,d ms (%.2f records/second).", (reportEndTime - jobStartTime), 1000d * (recordCount / (.001d + (reportEndTime - jobStartTime)))));
runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount);
}
catch(Exception e)
{

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java
View File

@ -66,7 +66,7 @@ public class StreamedETLProcess
.withField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING))
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)))
.withOutputMetaData(new QFunctionOutputMetaData()
.addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));
.withField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));
return new QProcessMetaData()
.withName(PROCESS_NAME)

102
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java Normal file
View File

@ -0,0 +1,102 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
/*******************************************************************************
** Base class for the Extract logic of Streamed ETL processes.
**
** These steps are invoked by both the "preview" and the "execute" steps of a
** StreamedETLWithFrontend process.
**
** Key here, is that subclasses here should put records that they're "Extracting"
** into the recordPipe member. That is to say, DO NOT use the recordList in
** the Step input/output objects.
**
** Ideally, they'll also stop once they've hit the "limit" number of records
** (though if you keep going, the pipe will get terminated and the job will be
** cancelled, etc...).
*******************************************************************************/
public abstract class AbstractExtractStep implements BackendStep
{
private RecordPipe recordPipe;
private Integer limit;
/*******************************************************************************
**
*******************************************************************************/
public Integer doCount(RunBackendStepInput runBackendStepInput) throws QException
{
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
public void setRecordPipe(RecordPipe recordPipe)
{
this.recordPipe = recordPipe;
}
/*******************************************************************************
**
*******************************************************************************/
public RecordPipe getRecordPipe()
{
return recordPipe;
}
/*******************************************************************************
** Getter for limit
**
*******************************************************************************/
public Integer getLimit()
{
return limit;
}
/*******************************************************************************
** Setter for limit
**
*******************************************************************************/
public void setLimit(Integer limit)
{
this.limit = limit;
}
}

105
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java Normal file
View File

@ -0,0 +1,105 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
/*******************************************************************************
** Base class for the Load (aka, store) logic of Streamed ETL processes.
**
** Records are to be read out of the input object's Records field, and after storing,
** should be written to the output object's Records, noting that when running
** as a streamed-ETL process, those input & output objects will be instances of
** the StreamedBackendStep{Input,Output} classes, that will be associated with
** a page of records flowing thorugh a pipe.
**
** Also - use the transaction member variable!!!
*******************************************************************************/
public abstract class AbstractLoadStep implements BackendStep
{
private Optional<QBackendTransaction> transaction = Optional.empty();
/*******************************************************************************
** Allow subclasses to do an action before the run is complete - before any
** pages of records are passed in.
*******************************************************************************/
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////
// noop in base class //
////////////////////////
}
/*******************************************************************************
** Allow subclasses to do an action after the run is complete - after the last
** page of records is passed in.
*******************************************************************************/
public void postRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////
// noop in base class //
////////////////////////
}
/*******************************************************************************
**
*******************************************************************************/
public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
return (Optional.empty());
}
/*******************************************************************************
** Setter for transaction
**
*******************************************************************************/
public void setTransaction(Optional<QBackendTransaction> transaction)
{
this.transaction = transaction;
}
/*******************************************************************************
** Getter for transaction
**
*******************************************************************************/
public Optional<QBackendTransaction> getTransaction()
{
return (transaction);
}
}

65
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java
View File

@ -19,56 +19,51 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for BulkDeleteStoreStep
** Base class for the Transform logic of Streamed ETL processes.
**
** Records are to be read out of the input object's Records field, and after storing,
** should be written to the output object's Records, noting that when running
** as a streamed-ETL process, those input & output objects will be instances of
** the StreamedBackendStep{Input,Output} classes, that will be associated with
** a page of records flowing through a pipe.
**
*******************************************************************************/
class BulkDeleteStoreStepTest
public abstract class AbstractTransformStep implements BackendStep, ProcessSummaryProviderInterface
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithoutFilter() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.setRecords(TestUtils.queryTable(TestUtils.defineTablePerson().getName()));
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkDeleteStoreStep().run(stepInput, stepOutput);
assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
/*******************************************************************************
** Allow subclasses to do an action before the run is complete - before any
** pages of records are passed in.
*******************************************************************************/
public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////
// noop in base class //
////////////////////////
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithFilter() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue("queryFilterJSON", JsonUtils.toJson(new QQueryFilter()));
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkDeleteStoreStep().run(stepInput, stepOutput);
assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
/*******************************************************************************
** Allow subclasses to do an action after the run is complete - after the last
** page of records is passed in.
*******************************************************************************/
public void postRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////
// noop in base class //
////////////////////////
}
}
}

116
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java Normal file
View File

@ -0,0 +1,116 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Base class for the StreamedETL preview & execute steps
*******************************************************************************/
public class BaseStreamedETLStep
{
private static final Logger LOG = LogManager.getLogger(BaseStreamedETLStep.class);
protected static final int PROCESS_OUTPUT_RECORD_LIST_LIMIT = 20;
/*******************************************************************************
**
*******************************************************************************/
protected AbstractExtractStep getExtractStep(RunBackendStepInput runBackendStepInput)
{
QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_EXTRACT_CODE);
return (QCodeLoader.getBackendStep(AbstractExtractStep.class, codeReference));
}
/*******************************************************************************
**
*******************************************************************************/
protected AbstractTransformStep getTransformStep(RunBackendStepInput runBackendStepInput)
{
QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE);
return (QCodeLoader.getBackendStep(AbstractTransformStep.class, codeReference));
}
/*******************************************************************************
**
*******************************************************************************/
protected AbstractLoadStep getLoadStep(RunBackendStepInput runBackendStepInput)
{
QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_LOAD_CODE);
return (QCodeLoader.getBackendStep(AbstractLoadStep.class, codeReference));
}
/*******************************************************************************
**
*******************************************************************************/
protected void updateRecordsWithDisplayValuesAndPossibleValues(RunBackendStepInput input, List<QRecord> list)
{
String destinationTable = input.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE);
QTableMetaData table = input.getInstance().getTable(destinationTable);
if(table != null && list != null)
{
QValueFormatter qValueFormatter = new QValueFormatter();
qValueFormatter.setDisplayValuesInRecords(table, list);
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(input.getInstance(), input.getSession());
qPossibleValueTranslator.translatePossibleValuesInRecords(input.getTable(), list);
}
}
/*******************************************************************************
**
*******************************************************************************/
protected void moveReviewStepAfterValidateStep(RunBackendStepOutput runBackendStepOutput)
{
LOG.info("Skipping to validation step");
ArrayList<String> stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList());
LOG.debug("Step list pre: " + stepList);
stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW));
stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW);
runBackendStepOutput.getProcessState().setStepList(stepList);
LOG.debug("Step list post: " + stepList);
}
}

179
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java Normal file
View File

@ -0,0 +1,179 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Generic implementation of an ExtractStep - that runs a Query action for a
** specified table.
**
** If a query is specified from the caller (e.g., using the process Callback
** mechanism), that will be used. Else a filter (object or json) in
** StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER will be checked.
*******************************************************************************/
public class ExtractViaQueryStep extends AbstractExtractStep
{
public static final String FIELD_SOURCE_TABLE = "sourceTable";
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance());
queryInput.setSession(runBackendStepInput.getSession());
queryInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
queryInput.setFilter(getQueryFilter(runBackendStepInput));
queryInput.setRecordPipe(getRecordPipe());
queryInput.setLimit(getLimit());
queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
new QueryAction().execute(queryInput);
///////////////////////////////////////////////////////////////////
// output is done into the pipe - so, nothing for us to do here. //
///////////////////////////////////////////////////////////////////
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Integer doCount(RunBackendStepInput runBackendStepInput) throws QException
{
CountInput countInput = new CountInput(runBackendStepInput.getInstance());
countInput.setSession(runBackendStepInput.getSession());
countInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
countInput.setFilter(getQueryFilter(runBackendStepInput));
CountOutput countOutput = new CountAction().execute(countInput);
return (countOutput.getCount());
}
/*******************************************************************************
**
*******************************************************************************/
protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException
{
String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson");
Serializable defaultQueryFilter = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER);
//////////////////////////////////////////////////////////////////////////////////////
// if the queryFilterJson field is populated, read the filter from it and return it //
//////////////////////////////////////////////////////////////////////////////////////
if(queryFilterJson != null)
{
return getQueryFilterFromJson(queryFilterJson, "Error loading query filter from json field");
}
else if(runBackendStepInput.getCallback() != null && runBackendStepInput.getCallback().getQueryFilter() != null)
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// else, try to get filter from process callback. if we got one, store it as a process value for later steps //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
QQueryFilter queryFilter = runBackendStepInput.getCallback().getQueryFilter();
runBackendStepInput.addValue("queryFilterJson", JsonUtils.toJson(queryFilter));
return (queryFilter);
}
else if(defaultQueryFilter instanceof QQueryFilter filter)
{
/////////////////////////////////////////////////////////////////////////////
// else, see if a defaultQueryFilter was specified as a QueryFilter object //
/////////////////////////////////////////////////////////////////////////////
return (filter);
}
else if(defaultQueryFilter instanceof String string)
{
/////////////////////////////////////////////////////////////////////////////
// else, see if a defaultQueryFilter was specified as a JSON string
/////////////////////////////////////////////////////////////////////////////
return getQueryFilterFromJson(string, "Error loading default query filter from json");
}
else if(StringUtils.hasContent(runBackendStepInput.getValueString("filterJSON")))
{
///////////////////////////////////////////////////////////////////////
// else, check for filterJSON from a frontend launching of a process //
///////////////////////////////////////////////////////////////////////
return getQueryFilterFromJson(runBackendStepInput.getValueString("filterJSON"), "Error loading default query filter from json");
}
else if(StringUtils.hasContent(runBackendStepInput.getValueString("recordIds")))
{
//////////////////////////////////////////////////////////////////////
// else, check for recordIds from a frontend launching of a process //
//////////////////////////////////////////////////////////////////////
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
if(table == null)
{
throw (new QException("source table name was not set - could not load records by id"));
}
String recordIds = runBackendStepInput.getValueString("recordIds");
Serializable[] split = recordIds.split(",");
List<Serializable> idStrings = Arrays.stream(split).toList();
return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, idStrings)));
}
throw (new QException("Could not find query filter for Extract step."));
}
/*******************************************************************************
**
*******************************************************************************/
private QQueryFilter getQueryFilterFromJson(String queryFilterJson, String message) throws QException
{
try
{
return (JsonUtils.toObject(queryFilterJson, QQueryFilter.class));
}
catch(IOException e)
{
throw new QException(message, e);
}
}
}

80
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java Normal file
View File

@ -0,0 +1,80 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.Optional;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Generic implementation of a LoadStep - that runs a Delete action for a
** specified table.
*******************************************************************************/
public class LoadViaDeleteStep extends AbstractLoadStep
{
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
QTableMetaData table = runBackendStepInput.getTable();
DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance());
deleteInput.setSession(runBackendStepInput.getSession());
deleteInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
deleteInput.setPrimaryKeys(runBackendStepInput.getRecords().stream().map(r -> r.getValue(table.getPrimaryKeyField())).collect(Collectors.toList()));
// todo? can make more efficient deletes, maybe? deleteInput.setQueryFilter();
getTransaction().ifPresent(deleteInput::setTransaction);
new DeleteAction().execute(deleteInput);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance());
insertInput.setSession(runBackendStepInput.getSession());
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(new InsertAction().openTransaction(insertInput)));
}
}

42
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStep.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java
View File

@ -19,41 +19,57 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Backend step to store the records from a bulk insert file
** Generic implementation of a LoadStep - that runs an Insert action for a
** specified table.
*******************************************************************************/
public class BulkInsertStoreRecordsStep implements BackendStep
public class LoadViaInsertStep extends AbstractLoadStep
{
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
List<QRecord> qRecords = BulkInsertUtils.getQRecordsFromFile(runBackendStepInput);
InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance());
insertInput.setSession(runBackendStepInput.getSession());
insertInput.setTableName(runBackendStepInput.getTableName());
insertInput.setRecords(qRecords);
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
insertInput.setRecords(runBackendStepInput.getRecords());
getTransaction().ifPresent(insertInput::setTransaction);
InsertOutput insertOutput = new InsertAction().execute(insertInput);
runBackendStepOutput.getRecords().addAll(insertOutput.getRecords());
}
InsertAction insertAction = new InsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
runBackendStepOutput.setRecords(insertOutput.getRecords());
/*******************************************************************************
**
*******************************************************************************/
@Override
public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance());
insertInput.setSession(runBackendStepInput.getSession());
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(new InsertAction().openTransaction(insertInput)));
}
}

77
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java Normal file
View File

@ -0,0 +1,77 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
/*******************************************************************************
** Generic implementation of a LoadStep - that runs an Update action for a
** specified table.
*******************************************************************************/
public class LoadViaUpdateStep extends AbstractLoadStep
{
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance());
updateInput.setSession(runBackendStepInput.getSession());
updateInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
updateInput.setRecords(runBackendStepInput.getRecords());
getTransaction().ifPresent(updateInput::setTransaction);
UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
runBackendStepOutput.getRecords().addAll(updateOutput.getRecords());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance());
insertInput.setSession(runBackendStepInput.getSession());
insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
return (Optional.of(new InsertAction().openTransaction(insertInput)));
}
}

40
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java Normal file
View File

@ -0,0 +1,40 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
/*******************************************************************************
** Interface for a class that can proivate a ProcessSummary - a list of Process Summary Lines
*******************************************************************************/
public interface ProcessSummaryProviderInterface
{
/*******************************************************************************
** Note - object needs to be serializable, and List isn't... so, use ArrayList?
*******************************************************************************/
ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen);
}

77
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepInput.java Normal file
View File

@ -0,0 +1,77 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Subclass of RunBackendStepInput, meant for use in the pseudo-steps used by
** the Streamed-ETL-with-frontend processes - where the Record list is not the
** full process's record list - rather - is just a page at a time -- so this class
** overrides the getRecords and setRecords method, to just work with that page.
**
** Note - of importance over time may be the RunBackendStepInput::cloneFieldsInto
** method - e.g., if new fields are added to that class!
*******************************************************************************/
public class StreamedBackendStepInput extends RunBackendStepInput
{
private List<QRecord> inputRecords;
/*******************************************************************************
**
*******************************************************************************/
public StreamedBackendStepInput(RunBackendStepInput runBackendStepInput, List<QRecord> inputRecords)
{
super(runBackendStepInput.getInstance());
runBackendStepInput.cloneFieldsInto(this);
this.inputRecords = inputRecords;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void setRecords(List<QRecord> records)
{
this.inputRecords = records;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> getRecords()
{
return (inputRecords);
}
}

74
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepOutput.java Normal file
View File

@ -0,0 +1,74 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Subclass of RunBackendStepOutput, meant for use in the pseudo-steps used by
** the Streamed-ETL-with-frontend processes - where the Record list is not the
** full process's record list - rather - is just a page at a time -- so this class
** overrides the getRecords and setRecords method, to just work with that page.
*******************************************************************************/
public class StreamedBackendStepOutput extends RunBackendStepOutput
{
private List<QRecord> outputRecords = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
public StreamedBackendStepOutput(RunBackendStepOutput runBackendStepOutput)
{
super();
setValues(runBackendStepOutput.getValues());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void setRecords(List<QRecord> records)
{
this.outputRecords = records;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> getRecords()
{
return (outputRecords);
}
}

179
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java Normal file
View File

@ -0,0 +1,179 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Backend step to do the execute portion of a streamed ETL job.
**
** Works within a transaction (per the backend module of the destination table).
*******************************************************************************/
public class StreamedETLExecuteStep extends BaseStreamedETLStep implements BackendStep
{
private int currentRowCount = 1;
/*******************************************************************************
**
*******************************************************************************/
@Override
@SuppressWarnings("checkstyle:indentation")
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Optional<QBackendTransaction> transaction = Optional.empty();
try
{
///////////////////////////////////////////////////////
// set up the extract, transform, and load functions //
///////////////////////////////////////////////////////
RecordPipe recordPipe = new RecordPipe();
AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
extractStep.setRecordPipe(recordPipe);
AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
AbstractLoadStep loadStep = getLoadStep(runBackendStepInput);
transformStep.preRun(runBackendStepInput, runBackendStepOutput);
loadStep.preRun(runBackendStepInput, runBackendStepOutput);
transaction = loadStep.openTransaction(runBackendStepInput);
loadStep.setTransaction(transaction);
List<QRecord> loadedRecordList = new ArrayList<>();
int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Execute>ExtractStep", null, recordPipe, (status) ->
{
extractStep.run(runBackendStepInput, runBackendStepOutput);
return (runBackendStepOutput);
},
() -> (consumeRecordsFromPipe(recordPipe, transformStep, loadStep, runBackendStepInput, runBackendStepOutput, loadedRecordList))
);
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, recordCount);
updateRecordsWithDisplayValuesAndPossibleValues(runBackendStepInput, loadedRecordList);
runBackendStepOutput.setRecords(loadedRecordList);
////////////////////////////////////////////////////////////////////////////////////////////////////
// get the process summary from the ... transform step? the load step? each knows some... todo? //
////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, transformStep.getProcessSummary(true));
transformStep.postRun(runBackendStepInput, runBackendStepOutput);
loadStep.postRun(runBackendStepInput, runBackendStepOutput);
/////////////////////
// commit the work //
/////////////////////
if(transaction.isPresent())
{
transaction.get().commit();
}
}
catch(Exception e)
{
////////////////////////////////////////////////////////////////////////////////
// rollback the work, then re-throw the error for up-stream to catch & report //
////////////////////////////////////////////////////////////////////////////////
if(transaction.isPresent())
{
transaction.get().rollback();
}
throw (e);
}
finally
{
////////////////////////////////////////////////////////////
// always close our transactions (e.g., jdbc connections) //
////////////////////////////////////////////////////////////
if(transaction.isPresent())
{
transaction.get().close();
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, AbstractLoadStep loadStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<QRecord> loadedRecordList) throws QException
{
Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT);
if(totalRows != null)
{
runBackendStepInput.getAsyncJobCallback().updateStatus(currentRowCount, totalRows);
}
///////////////////////////////////
// get the records from the pipe //
///////////////////////////////////
List<QRecord> qRecords = recordPipe.consumeAvailableRecords();
///////////////////////////////////////////////////////////////////////
// make streamed input & output objects from the run input & outputs //
///////////////////////////////////////////////////////////////////////
StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords);
StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput);
/////////////////////////////////////////////////////
// pass the records through the transform function //
/////////////////////////////////////////////////////
transformStep.run(streamedBackendStepInput, streamedBackendStepOutput);
////////////////////////////////////////////////
// pass the records through the load function //
////////////////////////////////////////////////
streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, streamedBackendStepOutput.getRecords());
streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput);
loadStep.run(streamedBackendStepInput, streamedBackendStepOutput);
///////////////////////////////////////////////////////
// copy a small number of records to the output list //
///////////////////////////////////////////////////////
int i = 0;
while(loadedRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < streamedBackendStepOutput.getRecords().size())
{
loadedRecordList.add(streamedBackendStepOutput.getRecords().get(i++));
}
currentRowCount += qRecords.size();
return (qRecords.size());
}
}

173
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java Normal file
View File

@ -0,0 +1,173 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Backend step to do a preview of a full streamed ETL job
*******************************************************************************/
public class StreamedETLPreviewStep extends BaseStreamedETLStep implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(StreamedETLPreviewStep.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
@SuppressWarnings("checkstyle:indentation")
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Integer limit = PROCESS_OUTPUT_RECORD_LIST_LIMIT; // todo - use a field instead of hard-coded here?
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the do-full-validation flag has already been set, then do the validation step instead of this one //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
boolean supportsFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION);
boolean doFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION);
if(supportsFullValidation && doFullValidation)
{
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
moveReviewStepAfterValidateStep(runBackendStepOutput);
return;
}
/////////////////////////////////////////////////////////////////
// if we're running inside an automation, then skip this step. //
/////////////////////////////////////////////////////////////////
if(runningWithinAutomation())
{
LOG.info("Skipping preview step when [" + runBackendStepInput.getProcessName() + "] is running as part of an automation.");
return;
}
///////////////////////////////////////////
// request a count from the extract step //
///////////////////////////////////////////
AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
Integer recordCount = extractStep.doCount(runBackendStepInput);
runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the count is less than the normal limit here, and this process supports validation, then go straight to the validation step //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// todo - maybe some future version we do this - maybe based on a user-preference
// if(supportsFullValidation && recordCount <= limit)
// {
// runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
// moveReviewStepAfterValidateStep(runBackendStepOutput);
// return;
// }
////////////////////////////////////////////////////////
// proceed with a doing a limited extract & transform //
////////////////////////////////////////////////////////
RecordPipe recordPipe = new RecordPipe();
extractStep.setLimit(limit);
extractStep.setRecordPipe(recordPipe);
AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
transformStep.preRun(runBackendStepInput, runBackendStepOutput);
List<QRecord> previewRecordList = new ArrayList<>();
new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) ->
{
runBackendStepInput.setAsyncJobCallback(status);
extractStep.run(runBackendStepInput, runBackendStepOutput);
return (runBackendStepOutput);
},
() -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, previewRecordList))
);
updateRecordsWithDisplayValuesAndPossibleValues(runBackendStepInput, previewRecordList);
runBackendStepOutput.setRecords(previewRecordList);
transformStep.postRun(runBackendStepInput, runBackendStepOutput);
}
/*******************************************************************************
**
*******************************************************************************/
private boolean runningWithinAutomation()
{
Exception e = new Exception();
for(StackTraceElement stackTraceElement : e.getStackTrace())
{
String className = stackTraceElement.getClassName();
if(className.contains("com.kingsrook.qqq.backend.core.actions.automation"))
{
return (true);
}
}
return false;
}
/*******************************************************************************
**
*******************************************************************************/
private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<QRecord> transformedRecordList) throws QException
{
///////////////////////////////////
// get the records from the pipe //
///////////////////////////////////
List<QRecord> qRecords = recordPipe.consumeAvailableRecords();
///////////////////////////////////////////////////////////////////////
// make streamed input & output objects from the run input & outputs //
///////////////////////////////////////////////////////////////////////
StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords);
StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput);
/////////////////////////////////////////////////////
// pass the records through the transform function //
/////////////////////////////////////////////////////
transformStep.run(streamedBackendStepInput, streamedBackendStepOutput);
////////////////////////////////////////////////////
// add the transformed records to the output list //
////////////////////////////////////////////////////
transformedRecordList.addAll(streamedBackendStepOutput.getRecords());
return (qRecords.size());
}
}

155
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java Normal file
View File

@ -0,0 +1,155 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Backend step to do a full validation of a streamed ETL job
*******************************************************************************/
public class StreamedETLValidateStep extends BaseStreamedETLStep implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(StreamedETLValidateStep.class);
private int currentRowCount = 1;
/*******************************************************************************
**
*******************************************************************************/
@Override
@SuppressWarnings("checkstyle:indentation")
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
/////////////////////////////////////////////////////////////////////
// check if we are supported in this process - if not, return noop //
/////////////////////////////////////////////////////////////////////
boolean supportsFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION);
if(!supportsFullValidation)
{
LOG.info("Process does not support validation, so skipping validation step");
return;
}
////////////////////////////////////////////////////////////////////////////////
// check if we've been requested to run in this process - if not, return noop //
////////////////////////////////////////////////////////////////////////////////
boolean doFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION);
if(!doFullValidation)
{
LOG.info("Not requested to do full validation, so skipping validation step");
return;
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we're proceeding with full validation, make sure the review step is after validation in the step list //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
moveReviewStepAfterValidateStep(runBackendStepOutput);
//////////////////////////////////////////////////////////
// basically repeat the preview step, but with no limit //
//////////////////////////////////////////////////////////
RecordPipe recordPipe = new RecordPipe();
AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
extractStep.setLimit(null);
extractStep.setRecordPipe(recordPipe);
AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
transformStep.preRun(runBackendStepInput, runBackendStepOutput);
List<QRecord> previewRecordList = new ArrayList<>();
int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Preview>ValidateStep", null, recordPipe, (status) ->
{
extractStep.run(runBackendStepInput, runBackendStepOutput);
return (runBackendStepOutput);
},
() -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, previewRecordList))
);
updateRecordsWithDisplayValuesAndPossibleValues(runBackendStepInput, previewRecordList);
runBackendStepOutput.setRecords(previewRecordList);
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, recordCount);
//////////////////////////////////////////////////////
// get the process summary from the validation step //
//////////////////////////////////////////////////////
runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, transformStep.getProcessSummary(false));
transformStep.postRun(runBackendStepInput, runBackendStepOutput);
}
/*******************************************************************************
**
*******************************************************************************/
private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<QRecord> previewRecordList) throws QException
{
Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT);
if(totalRows != null)
{
runBackendStepInput.getAsyncJobCallback().updateStatus(currentRowCount, totalRows);
}
///////////////////////////////////
// get the records from the pipe //
///////////////////////////////////
List<QRecord> qRecords = recordPipe.consumeAvailableRecords();
///////////////////////////////////////////////////////////////////////
// make streamed input & output objects from the run input & outputs //
///////////////////////////////////////////////////////////////////////
StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords);
StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput);
/////////////////////////////////////////////////////
// pass the records through the transform function //
/////////////////////////////////////////////////////
transformStep.run(streamedBackendStepInput, streamedBackendStepOutput);
///////////////////////////////////////////////////////
// copy a small number of records to the output list //
///////////////////////////////////////////////////////
int i = 0;
while(previewRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < streamedBackendStepOutput.getRecords().size())
{
previewRecordList.add(streamedBackendStepOutput.getRecords().get(i++));
}
currentRowCount += qRecords.size();
return (qRecords.size());
}
}

171
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java Normal file
View File

@ -0,0 +1,171 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
/*******************************************************************************
** Definition for Streamed ETL process that includes a frontend.
**
** This process uses 3 backend steps, and 2 frontend steps, as follows:
** - preview (backend) - does just a little work (limited # of rows), to give the
** user a preview of what the final result will be - e.g., some data to seed the review screen
** - review (frontend) - a review screen
** - validate (backend) - optionally (per input on review screen), does like the preview step,
** but on all records from the extract step.
** - execute (backend) - processes all the rows, does all the work.
** - result (frontend) - a result screen
**
** The preview & execute steps use additional BackendStep codes:
** - Extract - gets the rows to be processed. Used in preview (but only for a
** limited number of rows), and execute (without limit)
** - Transform - do whatever transformation is needed to the rows. Done on preview
** and execute. Always works with a "page" of records at a time.
** - Load - store the records into the backend, as appropriate. Always works
** with a "page" of records at a time. Only called by execute step.
*******************************************************************************/
public class StreamedETLWithFrontendProcess
{
public static final String STEP_NAME_PREVIEW = "preview";
public static final String STEP_NAME_REVIEW = "review";
public static final String STEP_NAME_VALIDATE = "validate";
public static final String STEP_NAME_EXECUTE = "execute";
public static final String STEP_NAME_RESULT = "result";
public static final String FIELD_EXTRACT_CODE = "extract"; // QCodeReference, of AbstractExtractStep
public static final String FIELD_TRANSFORM_CODE = "transform"; // QCodeReference, of AbstractTransformStep
public static final String FIELD_LOAD_CODE = "load"; // QCodeReference, of AbstractLoadStep
public static final String FIELD_SOURCE_TABLE = "sourceTable"; // String
public static final String FIELD_DESTINATION_TABLE = "destinationTable"; // String
public static final String FIELD_RECORD_COUNT = "recordCount"; // Integer
public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter)
public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean
public static final String FIELD_DO_FULL_VALIDATION = "doFullValidation"; // Boolean
public static final String FIELD_VALIDATION_SUMMARY = "validationSummary"; // List<ProcessSummaryLine>
public static final String FIELD_PROCESS_SUMMARY = "processResults"; // List<ProcessSummaryLine>
public static final String DEFAULT_PREVIEW_MESSAGE_FOR_INSERT = "This is a preview of the records that will be created.";
public static final String DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE = "This is a preview of the records that will be updated.";
public static final String DEFAULT_PREVIEW_MESSAGE_FOR_DELETE = "This is a preview of the records that will be deleted.";
public static final String FIELD_PREVIEW_MESSAGE = "previewMessage";
/*******************************************************************************
**
*******************************************************************************/
public static QProcessMetaData defineProcessMetaData(
String sourceTableName,
String destinationTableName,
Class<? extends AbstractExtractStep> extractStepClass,
Class<? extends AbstractTransformStep> transformStepClass,
Class<? extends AbstractLoadStep> loadStepClass
)
{
Map<String, Serializable> defaultFieldValues = new HashMap<>();
defaultFieldValues.put(FIELD_SOURCE_TABLE, sourceTableName);
defaultFieldValues.put(FIELD_DESTINATION_TABLE, destinationTableName);
return defineProcessMetaData(extractStepClass, transformStepClass, loadStepClass, defaultFieldValues);
}
/*******************************************************************************
** @param defaultFieldValues - expected to possibly contain values for the following field names:
** - FIELD_SOURCE_TABLE
** - FIELD_DESTINATION_TABLE
** - FIELD_SUPPORTS_FULL_VALIDATION
** - FIELD_DEFAULT_QUERY_FILTER
** - FIELD_DO_FULL_VALIDATION
** - FIELD_PREVIEW_MESSAGE
*******************************************************************************/
public static QProcessMetaData defineProcessMetaData(
Class<? extends AbstractExtractStep> extractStepClass,
Class<? extends AbstractTransformStep> transformStepClass,
Class<? extends AbstractLoadStep> loadStepClass,
Map<String, Serializable> defaultFieldValues
)
{
QStepMetaData previewStep = new QBackendStepMetaData()
.withName(STEP_NAME_PREVIEW)
.withCode(new QCodeReference(StreamedETLPreviewStep.class))
.withInputData(new QFunctionInputMetaData()
.withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE)))
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE)))
.withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, true)))
.withField(new QFieldMetaData(FIELD_DEFAULT_QUERY_FILTER, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DEFAULT_QUERY_FILTER)))
.withField(new QFieldMetaData(FIELD_EXTRACT_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(extractStepClass)))
.withField(new QFieldMetaData(FIELD_TRANSFORM_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(transformStepClass)))
.withField(new QFieldMetaData(FIELD_PREVIEW_MESSAGE, QFieldType.STRING).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_PREVIEW_MESSAGE, DEFAULT_PREVIEW_MESSAGE_FOR_INSERT)))
);
QFrontendStepMetaData reviewStep = new QFrontendStepMetaData()
.withName(STEP_NAME_REVIEW)
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VALIDATION_REVIEW_SCREEN));
QStepMetaData validateStep = new QBackendStepMetaData()
.withName(STEP_NAME_VALIDATE)
.withCode(new QCodeReference(StreamedETLValidateStep.class))
.withInputData(new QFunctionInputMetaData()
.withField(new QFieldMetaData(FIELD_DO_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.get(FIELD_DO_FULL_VALIDATION))))
.withOutputMetaData(new QFunctionOutputMetaData()
.withField(new QFieldMetaData(FIELD_VALIDATION_SUMMARY, QFieldType.STRING))
);
QStepMetaData executeStep = new QBackendStepMetaData()
.withName(STEP_NAME_EXECUTE)
.withCode(new QCodeReference(StreamedETLExecuteStep.class))
.withInputData(new QFunctionInputMetaData()
.withField(new QFieldMetaData(FIELD_LOAD_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(loadStepClass))))
.withOutputMetaData(new QFunctionOutputMetaData()
.withField(new QFieldMetaData(FIELD_PROCESS_SUMMARY, QFieldType.STRING))
);
QFrontendStepMetaData resultStep = new QFrontendStepMetaData()
.withName(STEP_NAME_RESULT)
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS));
return new QProcessMetaData()
.addStep(previewStep)
.addStep(reviewStep)
.addStep(validateStep)
.addStep(executeStep)
.addStep(resultStep);
}
}

4
qqq-backend-core/src/main/resources/log4j2.xml
View File

@ -2,11 +2,11 @@
<Configuration>
<Appenders>
<Console name="SystemOutAppender" target="SYSTEM_OUT">
<LevelRangeFilter minLevel="ERROR" maxLevel="all" onMatch="ACCEPT" onMismatch="DENY"/>
<LevelRangeFilter minLevel="ERROR" maxLevel="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%highlight{%date{ISO8601} | %relative | %level | %threadName | %logger{1} | %message%n}"/>
</Console>
<Syslog name="SyslogAppender" format="RFC5424" host="localhost" port="514" protocol="UDP" appName="qqq" facility="LOCAL0">
<LevelRangeFilter minLevel="ERROR" maxLevel="all" onMatch="ACCEPT" onMismatch="DENY"/>
<LevelRangeFilter minLevel="ERROR" maxLevel="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%date{ISO8601} | %relative | %level | %threadName | %logger{1} | %message%n"/>
</Syslog>
<File name="LogFileAppender" fileName="log/qqq.log">

Some files were not shown because too many files have changed in this diff Show More