QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-37 initial buidout of StreamedETLWithFrontendProcess

Browse Source
This commit is contained in:
darin.kelkhoff
2022-08-22 08:34:55 -05:00
parent 99f724e2c2
commit a86f42f373
22 changed files with 998 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
View File

@ -29,6 +29,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.interfaces.QActionInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -40,13 +43,18 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Base class for all core actions in the RDBMS module.
*******************************************************************************/
public abstract class AbstractRDBMSAction
public abstract class AbstractRDBMSAction implements QActionInterface
{
private static final Logger LOG = LogManager.getLogger(AbstractRDBMSAction.class);
/*******************************************************************************
** Get the table name to use in the RDBMS from a QTableMetaData.
@ -319,4 +327,26 @@ public abstract class AbstractRDBMSAction
{
return fieldType == QFieldType.STRING || fieldType == QFieldType.TEXT || fieldType == QFieldType.HTML || fieldType == QFieldType.PASSWORD;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException
{
try
{
LOG.info("Opening transaction");
Connection connection = getConnection(input);
return (new RDBMSTransaction(connection));
}
catch(Exception e)
{
throw new QException("Error opening transaction: " + e.getMessage(), e);
}
}
}

23
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
View File

@ -28,7 +28,6 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@ -160,26 +159,4 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QBackendTransaction openTransaction(InsertInput insertInput) throws QException
{
try
{
LOG.info("Opening transaction");
Connection connection = getConnection(insertInput);
return (new RDBMSTransaction(connection));
}
catch(Exception e)
{
throw new QException("Error opening transaction: " + e.getMessage(), e);
}
}
}

6
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
View File

@ -123,6 +123,12 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
}
queryOutput.addRecord(record);
if(queryInput.getAsyncJobCallback().wasCancelRequested())
{
LOG.info("Breaking query job, as requested.");
break;
}
}
}), params);

39
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java
View File

@ -114,14 +114,37 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
outputRecords.add(outputRecord);
}
try(Connection connection = getConnection(updateInput))
try
{
/////////////////////////////////////////////////////////////////////////////////////////////
// process each distinct list of fields being updated (e.g., each different SQL statement) //
/////////////////////////////////////////////////////////////////////////////////////////////
for(List<String> fieldsBeingUpdated : recordsByFieldBeingUpdated.keySet())
Connection connection;
boolean needToCloseConnection = false;
if(updateInput.getTransaction() != null && updateInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
{
updateRecordsWithMatchingListOfFields(updateInput, connection, table, recordsByFieldBeingUpdated.get(fieldsBeingUpdated), fieldsBeingUpdated);
LOG.debug("Using connection from insertInput [" + rdbmsTransaction.getConnection() + "]");
connection = rdbmsTransaction.getConnection();
}
else
{
connection = getConnection(updateInput);
needToCloseConnection = true;
}
try
{
/////////////////////////////////////////////////////////////////////////////////////////////
// process each distinct list of fields being updated (e.g., each different SQL statement) //
/////////////////////////////////////////////////////////////////////////////////////////////
for(List<String> fieldsBeingUpdated : recordsByFieldBeingUpdated.keySet())
{
updateRecordsWithMatchingListOfFields(updateInput, connection, table, recordsByFieldBeingUpdated.get(fieldsBeingUpdated), fieldsBeingUpdated);
}
}
finally
{
if(needToCloseConnection)
{
connection.close();
}
}
return rs;
@ -191,7 +214,6 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
/*******************************************************************************
**
*******************************************************************************/
@ -276,6 +298,8 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
@ -285,5 +309,4 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
updateInput.getAsyncJobCallback().updateStatus(statusCounter, updateInput.getRecords().size());
}
}