QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #45 from Kingsrook/feature/garbage-collector

Browse Source 
Feature/garbage collector
This commit is contained in:
darin.kelkhoff
2023-11-24 09:24:24 -06:00
committed by GitHub
parent 2b263204b8 118433178d
commit 50c9a3eef2
5 changed files with 816 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java
View File

@ -24,15 +24,18 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.utils;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -127,6 +130,16 @@ public class BackendQueryFilterUtils
@SuppressWarnings("checkstyle:indentation")
public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value)
{
ListIterator<Serializable> valueListIterator = criterion.getValues().listIterator();
while(valueListIterator.hasNext())
{
Serializable criteriaValue = valueListIterator.next();
if(criteriaValue instanceof AbstractFilterExpression<?> expression)
{
valueListIterator.set(expression.evaluate());
}
}
boolean criterionMatches = switch(criterion.getOperator())
{
case EQUALS -> testEquals(criterion, value);
@ -287,26 +300,15 @@ public class BackendQueryFilterUtils
if(b instanceof LocalDate || a instanceof LocalDate)
{
LocalDate valueDate;
if(b instanceof LocalDate ld)
{
valueDate = ld;
}
else
{
valueDate = ValueUtils.getValueAsLocalDate(b);
}
LocalDate criterionDate;
if(a instanceof LocalDate ld)
{
criterionDate = ld;
}
else
{
criterionDate = ValueUtils.getValueAsLocalDate(a);
}
LocalDate valueDate = ValueUtils.getValueAsLocalDate(b);
LocalDate criterionDate = ValueUtils.getValueAsLocalDate(a);
return (valueDate.isAfter(criterionDate));
}
if(b instanceof Instant || a instanceof Instant)
{
Instant valueDate = ValueUtils.getValueAsInstant(b);
Instant criterionDate = ValueUtils.getValueAsInstant(a);
return (valueDate.isAfter(criterionDate));
}

60
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java Normal file
View File

@ -0,0 +1,60 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
/*******************************************************************************
**
*******************************************************************************/
public class GarbageCollectorExtractStep extends ExtractViaQueryStep
{
/*******************************************************************************
**
*******************************************************************************/
@Override
protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException
{
//////////////////////////////////////////////////////////////////////////////////////////
// in case the process was executed via a frontend, and the user specified a limitDate, //
// then put that date in the defaultQueryFilter, rather than the default //
//////////////////////////////////////////////////////////////////////////////////////////
Instant limitDate = ValueUtils.getValueAsInstant(runBackendStepInput.getValue("limitDate"));
if(limitDate != null)
{
QQueryFilter defaultQueryFilter = (QQueryFilter) runBackendStepInput.getValue("defaultQueryFilter");
defaultQueryFilter.getCriteria().get(0).setValues(ListBuilder.of(limitDate));
}
return super.getQueryFilter(runBackendStepInput);
}
}

107
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorProcessMetaDataProducer.java Normal file
View File

@ -0,0 +1,107 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.LESS_THAN;
/*******************************************************************************
** Create a garbage collector process for a given table.
**
** Process will be named: tableName + "GarbageCollector"
**
** It requires a dateTime field which is used in the query to find old records
** to be deleted. This dateTime field is, by default, compared with the input
** 'nowWithOffset' (e.g., .minus(30, DAYS)).
**
** Child join tables can also be GC'ed. This behavior is controlled via the
** joinedTablesToAlsoDelete parameter, which behaves as follows:
** - if the value is "*", then ALL descendent joins are GC'ed from.
** - if the value is null, then NO descendent joins are GC'ed from.
** - else the value is split on commas, and only table names found in the split are GC'ed.
**
** The process is, by default, associated with its associated table, so it can
** show up in UI's if permissed as such. When ran in a UI, it presents a limitDate
** field, which users can use to override the default limit.
**
** It does not get a schedule by default.
**
*******************************************************************************/
public class GarbageCollectorProcessMetaDataProducer
{
/*******************************************************************************
** See class header for param descriptions.
*******************************************************************************/
public static QProcessMetaData createProcess(String tableName, String dateTimeField, NowWithOffset nowWithOffset, String joinedTablesToAlsoDelete)
{
QProcessMetaData processMetaData = StreamedETLWithFrontendProcess.processMetaDataBuilder()
.withName(tableName + "GarbageCollector")
.withIcon(new QIcon().withName("auto_delete"))
.withTableName(tableName)
.withSourceTable(tableName)
.withDestinationTable(tableName)
.withExtractStepClass(GarbageCollectorExtractStep.class)
.withTransformStepClass(GarbageCollectorTransformStep.class)
.withLoadStepClass(LoadViaDeleteStep.class)
.withTransactionLevelPage()
.withPreviewMessage(StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_DELETE)
.withReviewStepRecordFields(List.of(
new QFieldMetaData("id", QFieldType.INTEGER),
new QFieldMetaData(dateTimeField, QFieldType.DATE_TIME)
))
.withDefaultQueryFilter(new QQueryFilter(new QFilterCriteria(dateTimeField, LESS_THAN, nowWithOffset)))
.getProcessMetaData();
processMetaData.getBackendStep(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)
.withInputData(new QFunctionInputMetaData()
.withField(new QFieldMetaData("joinedTablesToAlsoDelete", QFieldType.STRING).withDefaultValue(joinedTablesToAlsoDelete)));
processMetaData.addStep(0, new QFrontendStepMetaData()
.withName("input")
.withLabel("Input")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text", """
You can specify a limit date, or let the system use its default.
"""))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
.withFormField(new QFieldMetaData("limitDate", QFieldType.DATE_TIME))
);
return (processMetaData);
}
}

300
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTransformStep.java Normal file
View File

@ -0,0 +1,300 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN;
/*******************************************************************************
**
*******************************************************************************/
public class GarbageCollectorTransformStep extends AbstractTransformStep
{
private static final QLogger LOG = QLogger.getLogger(GarbageCollectorTransformStep.class);
private int count = 0;
private int total = 0;
private final ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK)
.withMessageSuffix(" deleted")
.withSingularFutureMessage("will be")
.withPluralFutureMessage("will be")
.withSingularPastMessage("has been")
.withPluralPastMessage("have been");
private Map<String, Integer> descendantRecordCountToDelete = new LinkedHashMap<>();
/*******************************************************************************
** getProcessSummary
*
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
{
ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
okSummary.addSelfToListIfAnyCount(rs);
for(Map.Entry<String, Integer> entry : descendantRecordCountToDelete.entrySet())
{
ProcessSummaryLine childSummary = new ProcessSummaryLine(Status.OK)
.withMessageSuffix(" deleted")
.withSingularFutureMessage("associated " + entry.getKey() + " record will be")
.withPluralFutureMessage("associated " + entry.getKey() + " records will be")
.withSingularPastMessage("associated " + entry.getKey() + " record has been")
.withPluralPastMessage("associated " + entry.getKey() + " records have been");
childSummary.setCount(entry.getValue());
rs.add(childSummary);
}
if(total == 0)
{
rs.add(new ProcessSummaryLine(Status.INFO, null, "No records were found to be garbage collected."));
}
return (rs);
}
/*******************************************************************************
** run
*
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////////////
// return if no input records //
////////////////////////////////
if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords()))
{
return;
}
///////////////////////////////////////////////////////////////////
// keep a count (in case table doesn't support count capacility) //
///////////////////////////////////////////////////////////////////
count += runBackendStepInput.getRecords().size();
total = Objects.requireNonNullElse(runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT), count);
runBackendStepInput.getAsyncJobCallback().updateStatus("Validating records", count, total);
////////////////////////////////////////////////////////////////////////////////////////
// process the joinedTablesToAlsoDelete value. //
// if it's "*", interpret that as all tables in the instance. //
// else split it on commas. //
// note that absent value or empty string means we won't delete from any other tables //
////////////////////////////////////////////////////////////////////////////////////////
String joinedTablesToAlsoDelete = runBackendStepInput.getValueString("joinedTablesToAlsoDelete");
Set<String> setOfJoinedTablesToAlsoDelete = new HashSet<>();
if("*".equals(joinedTablesToAlsoDelete))
{
setOfJoinedTablesToAlsoDelete.addAll(QContext.getQInstance().getTables().keySet());
}
else if(joinedTablesToAlsoDelete != null)
{
setOfJoinedTablesToAlsoDelete.addAll(Arrays.asList(joinedTablesToAlsoDelete.split(",")));
}
///////////////////
// process joins //
///////////////////
String tableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
lookForJoins(runBackendStepInput, tableName, runBackendStepInput.getRecords(), new HashSet<>(Set.of(tableName)), setOfJoinedTablesToAlsoDelete);
LOG.info("GarbageCollector called with a page of records", logPair("count", runBackendStepInput.getRecords().size()), logPair("table", tableName));
////////////////////////////////////////////////////
// move records (from primary table) to next step //
////////////////////////////////////////////////////
for(QRecord qRecord : runBackendStepInput.getRecords())
{
okSummary.incrementCountAndAddPrimaryKey(qRecord.getValue(runBackendStepInput.getTable().getPrimaryKeyField()));
runBackendStepOutput.getRecords().add(qRecord);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void postRun(BackendStepPostRunInput runBackendStepInput, BackendStepPostRunOutput runBackendStepOutput) throws QException
{
super.postRun(runBackendStepInput, runBackendStepOutput);
///////////////////////////////////////////////////////////////////////////////////////
// if we've just finished the validate step - //
// and if there wasn't a COUNT performed (e.g., because the table didn't support it) //
// then set our total that we accumulated into the count field. //
///////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
{
if(runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null)
{
runBackendStepInput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, total);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private void lookForJoins(RunBackendStepInput runBackendStepInput, String tableName, List<QRecord> records, Set<String> visitedTables, Set<String> allowedToAlsoDelete) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////
// if we've already visited all the tables we're allowed to delete, then return early //
////////////////////////////////////////////////////////////////////////////////////////
HashSet<String> anyAllowedLeft = new HashSet<>(allowedToAlsoDelete);
anyAllowedLeft.removeAll(visitedTables);
if(CollectionUtils.nullSafeIsEmpty(anyAllowedLeft))
{
return;
}
QInstance qInstance = QContext.getQInstance();
JoinGraph joinGraph = qInstance.getJoinGraph();
////////////////////////////////////////////////////////////////////
// get join connections from this table from the joinGraph object //
////////////////////////////////////////////////////////////////////
Set<JoinGraph.JoinConnectionList> joinConnections = joinGraph.getJoinConnections(tableName);
for(JoinGraph.JoinConnectionList joinConnectionList : CollectionUtils.nonNullCollection(joinConnections))
{
List<JoinGraph.JoinConnection> list = joinConnectionList.list();
JoinGraph.JoinConnection joinConnection = list.get(0);
QJoinMetaData join = qInstance.getJoin(joinConnection.viaJoinName());
String recurOnTable = null;
String thisTableFKeyField = null;
String joinTablePrimaryKeyField = null;
////////////////////////////////////////////////////////////////////////////////////////////////////////////
// find the input table in the join - but only if it's on a '1' side of the join (not a many side) //
// this means we may get out of this if/else with recurOnTable = null, if we shouldn't process this join. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(join.getLeftTable().equals(tableName) && (join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.ONE_TO_ONE)))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if this table is on the left side of this join, and it's a 1-n or 1-1, then delete from the right table //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
recurOnTable = join.getRightTable();
thisTableFKeyField = join.getJoinOns().get(0).getLeftField();
joinTablePrimaryKeyField = join.getJoinOns().get(0).getRightField();
}
else if(join.getRightTable().equals(tableName) && (join.getType().equals(JoinType.MANY_TO_ONE) || join.getType().equals(JoinType.ONE_TO_ONE)))
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// else if this table is on the right side of this join, and it's n-1 or 1-1, then delete from the left table //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
recurOnTable = join.getLeftTable();
thisTableFKeyField = join.getJoinOns().get(0).getRightField();
joinTablePrimaryKeyField = join.getJoinOns().get(0).getLeftField();
}
//////////////////////////////////////////////////////////////////////////////////////////////
// if we found a table to 'recur' on, and we haven't visited it before, then process it now //
//////////////////////////////////////////////////////////////////////////////////////////////
if(recurOnTable != null && !visitedTables.contains(recurOnTable))
{
if(join.getJoinOns().size() > 1)
{
LOG.warn("We would delete child records from the join [" + join.getName() + "], but it has multiple joinOns, and we don't support that yet...");
continue;
}
visitedTables.add(recurOnTable);
////////////////////////////////////////////////////////////
// query for records in the child table based on the join //
////////////////////////////////////////////////////////////
QTableMetaData foreignTable = qInstance.getTable(recurOnTable);
String finalThisTableFKeyField = thisTableFKeyField;
List<Serializable> foreignKeys = records.stream().map(r -> r.getValue(finalThisTableFKeyField)).distinct().toList();
List<QRecord> foreignRecords = new QueryAction().execute(new QueryInput(recurOnTable).withFilter(new QQueryFilter(new QFilterCriteria(joinTablePrimaryKeyField, IN, foreignKeys)))).getRecords();
////////////////////////////////////////////////////////////////////////////////////
// make a recursive call looking for children of this table //
// we do this before we delete from this table, so that the children can be found //
////////////////////////////////////////////////////////////////////////////////////
lookForJoins(runBackendStepInput, recurOnTable, foreignRecords, visitedTables, allowedToAlsoDelete);
if(allowedToAlsoDelete.contains(recurOnTable))
{
LOG.info("Deleting descendant records from: " + recurOnTable);
descendantRecordCountToDelete.putIfAbsent(foreignTable.getLabel(), 0);
descendantRecordCountToDelete.put(foreignTable.getLabel(), descendantRecordCountToDelete.get(foreignTable.getLabel()) + foreignRecords.size());
/////////////////////////////////////////////////////////////////////
// if this is the execute step - then do it - delete the children. //
/////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE))
{
List<Serializable> foreignPKeys = foreignRecords.stream().map(r -> r.getValue(foreignTable.getPrimaryKeyField())).toList();
new DeleteAction().execute(new DeleteInput(recurOnTable).withPrimaryKeys(foreignPKeys).withTransaction(getTransaction().orElse(null)));
}
}
}
}
}
}

328
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTest.java Normal file
View File

@ -0,0 +1,328 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for GarbageCollectorTransformStep
*******************************************************************************/
class GarbageCollectorTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBasic() throws QException
{
QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_PERSON_MEMORY, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), null);
QContext.getQInstance().addProcess(process);
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords()));
RunProcessInput input = new RunProcessInput();
input.setProcessName(process.getName());
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
new RunProcessAction().execute(input);
QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
assertEquals(2, queryOutput.getRecords().size());
assertEquals(Set.of(4, 5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
}
/*******************************************************************************
**
*******************************************************************************/
private static List<QRecord> getPersonRecords()
{
List<QRecord> records = List.of(
new QRecord().withValue("id", 1).withValue("createDate", Instant.now().minus(90, ChronoUnit.DAYS)),
new QRecord().withValue("id", 2).withValue("createDate", Instant.now().minus(31, ChronoUnit.DAYS)),
new QRecord().withValue("id", 3).withValue("createDate", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
new QRecord().withValue("id", 4).withValue("createDate", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
new QRecord().withValue("id", 5).withValue("createDate", Instant.now().minus(5, ChronoUnit.DAYS)));
return records;
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOverrideDate() throws QException
{
QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_PERSON_MEMORY, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), null);
QContext.getQInstance().addProcess(process);
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords()));
///////////////////////////////////////////////////////////////
// run with a limit of 100 days ago, and 0 should be deleted //
///////////////////////////////////////////////////////////////
RunProcessInput input = new RunProcessInput();
input.setProcessName(process.getName());
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
input.addValue("limitDate", Instant.now().minus(100, ChronoUnit.DAYS));
new RunProcessAction().execute(input);
QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
assertEquals(5, queryOutput.getRecords().size());
///////////////////////////////////////////////////
// re-run with 10 days, and all but 1 be deleted //
///////////////////////////////////////////////////
input.addValue("limitDate", Instant.now().minus(10, ChronoUnit.DAYS));
new RunProcessAction().execute(input);
queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
assertEquals(1, queryOutput.getRecords().size());
///////////////////////////////////////////////
// re-run with 1 day, and all end up deleted //
///////////////////////////////////////////////
input.addValue("limitDate", Instant.now().minus(1, ChronoUnit.DAYS));
new RunProcessAction().execute(input);
queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
assertEquals(0, queryOutput.getRecords().size());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithDeleteAllJoins() throws QException
{
QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), "*");
QContext.getQInstance().addProcess(process);
QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_ORDER).withRecords(getOrderRecords()));
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM).withRecords(getLineItemRecords()));
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withRecords(getLineItemExtrinsicRecords()));
RunProcessInput input = new RunProcessInput();
input.setProcessName(process.getName());
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
new RunProcessAction().execute(input);
QueryOutput orderQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_ORDER).withFilter(new QQueryFilter()));
assertEquals(2, orderQueryOutput.getRecords().size());
assertEquals(Set.of(4, 5), orderQueryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
QueryOutput lineItemQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM).withFilter(new QQueryFilter()));
assertEquals(9, lineItemQueryOutput.getRecords().size());
assertEquals(Set.of(4, 5), lineItemQueryOutput.getRecords().stream().map(r -> r.getValueInteger("orderId")).collect(Collectors.toSet()));
QueryOutput lineItemExtrinsicQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withFilter(new QQueryFilter()));
assertEquals(5, lineItemExtrinsicQueryOutput.getRecords().size());
assertEquals(Set.of(7, 9, 11, 13, 15), lineItemExtrinsicQueryOutput.getRecords().stream().map(r -> r.getValueInteger("lineItemId")).collect(Collectors.toSet()));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithDeleteSomeJoins() throws QException
{
QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), TestUtils.TABLE_NAME_LINE_ITEM);
QContext.getQInstance().addProcess(process);
//////////////////////////////////////////////////////////////////////////
// remove table's associations - as they implicitly cascade the delete! //
//////////////////////////////////////////////////////////////////////////
QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER).withAssociations(new ArrayList<>());
QContext.getQInstance().getTable(TestUtils.TABLE_NAME_LINE_ITEM).withAssociations(new ArrayList<>());
QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_ORDER).withRecords(getOrderRecords()));
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM).withRecords(getLineItemRecords()));
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withRecords(getLineItemExtrinsicRecords()));
RunProcessInput input = new RunProcessInput();
input.setProcessName(process.getName());
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
new RunProcessAction().execute(input);
QueryOutput orderQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_ORDER).withFilter(new QQueryFilter()));
assertEquals(2, orderQueryOutput.getRecords().size());
assertEquals(Set.of(4, 5), orderQueryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
QueryOutput lineItemQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM).withFilter(new QQueryFilter()));
assertEquals(9, lineItemQueryOutput.getRecords().size());
assertEquals(Set.of(4, 5), lineItemQueryOutput.getRecords().stream().map(r -> r.getValueInteger("orderId")).collect(Collectors.toSet()));
QueryOutput lineItemExtrinsicQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withFilter(new QQueryFilter()));
assertEquals(8, lineItemExtrinsicQueryOutput.getRecords().size());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithDeleteNoJoins() throws QException
{
QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), null);
QContext.getQInstance().addProcess(process);
////////////////////////////////////////////////////////////////////////////////
// remove order table's associations - as they implicitly cascade the delete! //
////////////////////////////////////////////////////////////////////////////////
QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER).withAssociations(new ArrayList<>());
QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_ORDER).withRecords(getOrderRecords()));
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM).withRecords(getLineItemRecords()));
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withRecords(getLineItemExtrinsicRecords()));
RunProcessInput input = new RunProcessInput();
input.setProcessName(process.getName());
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
new RunProcessAction().execute(input);
QueryOutput orderQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_ORDER).withFilter(new QQueryFilter()));
assertEquals(2, orderQueryOutput.getRecords().size());
assertEquals(Set.of(4, 5), orderQueryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
QueryOutput lineItemQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM).withFilter(new QQueryFilter()));
assertEquals(15, lineItemQueryOutput.getRecords().size());
QueryOutput lineItemExtrinsicQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withFilter(new QQueryFilter()));
assertEquals(8, lineItemExtrinsicQueryOutput.getRecords().size());
}
/*******************************************************************************
**
*******************************************************************************/
private static List<QRecord> getOrderRecords()
{
List<QRecord> records = List.of(
new QRecord().withValue("id", 1).withValue("createDate", Instant.now().minus(90, ChronoUnit.DAYS)),
new QRecord().withValue("id", 2).withValue("createDate", Instant.now().minus(31, ChronoUnit.DAYS)),
new QRecord().withValue("id", 3).withValue("createDate", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
new QRecord().withValue("id", 4).withValue("createDate", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
new QRecord().withValue("id", 5).withValue("createDate", Instant.now().minus(5, ChronoUnit.DAYS)));
return records;
}
/*******************************************************************************
**
*******************************************************************************/
private static List<QRecord> getLineItemRecords()
{
List<QRecord> records = List.of(
new QRecord().withValue("id", 1).withValue("orderId", 1),
new QRecord().withValue("id", 2).withValue("orderId", 2),
new QRecord().withValue("id", 3).withValue("orderId", 2),
new QRecord().withValue("id", 4).withValue("orderId", 3),
new QRecord().withValue("id", 5).withValue("orderId", 3),
new QRecord().withValue("id", 6).withValue("orderId", 3),
new QRecord().withValue("id", 7).withValue("orderId", 4),
new QRecord().withValue("id", 8).withValue("orderId", 4),
new QRecord().withValue("id", 9).withValue("orderId", 4),
new QRecord().withValue("id", 10).withValue("orderId", 4),
new QRecord().withValue("id", 11).withValue("orderId", 5),
new QRecord().withValue("id", 12).withValue("orderId", 5),
new QRecord().withValue("id", 13).withValue("orderId", 5),
new QRecord().withValue("id", 14).withValue("orderId", 5),
new QRecord().withValue("id", 15).withValue("orderId", 5));
return records;
}
/*******************************************************************************
**
*******************************************************************************/
private static List<QRecord> getLineItemExtrinsicRecords()
{
List<QRecord> records = List.of(
new QRecord().withValue("id", 1).withValue("lineItemId", 1),
new QRecord().withValue("id", 2).withValue("lineItemId", 3),
new QRecord().withValue("id", 3).withValue("lineItemId", 5),
new QRecord().withValue("id", 4).withValue("lineItemId", 7),
new QRecord().withValue("id", 5).withValue("lineItemId", 9),
new QRecord().withValue("id", 6).withValue("lineItemId", 11),
new QRecord().withValue("id", 7).withValue("lineItemId", 13),
new QRecord().withValue("id", 8).withValue("lineItemId", 15));
return records;
}
}