diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java new file mode 100644 index 00000000..4c4830d5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class GarbageCollectorExtractStep extends ExtractViaQueryStep +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException + { + ////////////////////////////////////////////////////////////////////////////////////////// + // in case the process was executed via a frontend, and the user specified a limitDate, // + // then put that date in the defaultQueryFilter, rather than the default // + ////////////////////////////////////////////////////////////////////////////////////////// + Instant limitDate = ValueUtils.getValueAsInstant(runBackendStepInput.getValue("limitDate")); + if(limitDate != null) + { + QQueryFilter defaultQueryFilter = (QQueryFilter) runBackendStepInput.getValue("defaultQueryFilter"); + defaultQueryFilter.getCriteria().get(0).setValues(ListBuilder.of(limitDate)); + } + + return super.getQueryFilter(runBackendStepInput); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorProcessMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorProcessMetaDataProducer.java new file mode 100644 index 00000000..a9d28fac --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorProcessMetaDataProducer.java @@ -0,0 +1,107 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.LESS_THAN; + + +/******************************************************************************* + ** Create a garbage collector process for a given table. + ** + ** Process will be named: tableName + "GarbageCollector" + ** + ** It requires a dateTime field which is used in the query to find old records + ** to be deleted. This dateTime field is, by default, compared with the input + ** 'nowWithOffset' (e.g., .minus(30, DAYS)). + ** + ** Child join tables can also be GC'ed. This behavior is controlled via the + ** joinedTablesToAlsoDelete parameter, which behaves as follows: + ** - if the value is "*", then ALL descendent joins are GC'ed from. + ** - if the value is null, then NO descendent joins are GC'ed from. + ** - else the value is split on commas, and only table names found in the split are GC'ed. + ** + ** The process is, by default, associated with its associated table, so it can + ** show up in UI's if permissed as such. When ran in a UI, it presents a limitDate + ** field, which users can use to override the default limit. + ** + ** It does not get a schedule by default. + ** + *******************************************************************************/ +public class GarbageCollectorProcessMetaDataProducer +{ + + /******************************************************************************* + ** See class header for param descriptions. + *******************************************************************************/ + public static QProcessMetaData createProcess(String tableName, String dateTimeField, NowWithOffset nowWithOffset, String joinedTablesToAlsoDelete) + { + QProcessMetaData processMetaData = StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(tableName + "GarbageCollector") + .withIcon(new QIcon().withName("auto_delete")) + .withTableName(tableName) + .withSourceTable(tableName) + .withDestinationTable(tableName) + .withExtractStepClass(GarbageCollectorExtractStep.class) + .withTransformStepClass(GarbageCollectorTransformStep.class) + .withLoadStepClass(LoadViaDeleteStep.class) + .withTransactionLevelPage() + .withPreviewMessage(StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_DELETE) + .withReviewStepRecordFields(List.of( + new QFieldMetaData("id", QFieldType.INTEGER), + new QFieldMetaData(dateTimeField, QFieldType.DATE_TIME) + )) + .withDefaultQueryFilter(new QQueryFilter(new QFilterCriteria(dateTimeField, LESS_THAN, nowWithOffset))) + .getProcessMetaData(); + + processMetaData.getBackendStep(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData("joinedTablesToAlsoDelete", QFieldType.STRING).withDefaultValue(joinedTablesToAlsoDelete))); + + processMetaData.addStep(0, new QFrontendStepMetaData() + .withName("input") + .withLabel("Input") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text", """ + You can specify a limit date, or let the system use its default. + """)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) + .withFormField(new QFieldMetaData("limitDate", QFieldType.DATE_TIME)) + ); + + return (processMetaData); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTransformStep.java new file mode 100644 index 00000000..ef233112 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTransformStep.java @@ -0,0 +1,300 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class GarbageCollectorTransformStep extends AbstractTransformStep +{ + private static final QLogger LOG = QLogger.getLogger(GarbageCollectorTransformStep.class); + + private int count = 0; + private int total = 0; + + private final ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK) + .withMessageSuffix(" deleted") + .withSingularFutureMessage("will be") + .withPluralFutureMessage("will be") + .withSingularPastMessage("has been") + .withPluralPastMessage("have been"); + + private Map descendantRecordCountToDelete = new LinkedHashMap<>(); + + + + /******************************************************************************* + ** getProcessSummary + * + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + ArrayList rs = new ArrayList<>(); + okSummary.addSelfToListIfAnyCount(rs); + + for(Map.Entry entry : descendantRecordCountToDelete.entrySet()) + { + ProcessSummaryLine childSummary = new ProcessSummaryLine(Status.OK) + .withMessageSuffix(" deleted") + .withSingularFutureMessage("associated " + entry.getKey() + " record will be") + .withPluralFutureMessage("associated " + entry.getKey() + " records will be") + .withSingularPastMessage("associated " + entry.getKey() + " record has been") + .withPluralPastMessage("associated " + entry.getKey() + " records have been"); + childSummary.setCount(entry.getValue()); + rs.add(childSummary); + } + + if(total == 0) + { + rs.add(new ProcessSummaryLine(Status.INFO, null, "No records were found to be garbage collected.")); + } + + return (rs); + } + + + + /******************************************************************************* + ** run + * + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + //////////////////////////////// + // return if no input records // + //////////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords())) + { + return; + } + + /////////////////////////////////////////////////////////////////// + // keep a count (in case table doesn't support count capacility) // + /////////////////////////////////////////////////////////////////// + count += runBackendStepInput.getRecords().size(); + total = Objects.requireNonNullElse(runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT), count); + runBackendStepInput.getAsyncJobCallback().updateStatus("Validating records", count, total); + + //////////////////////////////////////////////////////////////////////////////////////// + // process the joinedTablesToAlsoDelete value. // + // if it's "*", interpret that as all tables in the instance. // + // else split it on commas. // + // note that absent value or empty string means we won't delete from any other tables // + //////////////////////////////////////////////////////////////////////////////////////// + String joinedTablesToAlsoDelete = runBackendStepInput.getValueString("joinedTablesToAlsoDelete"); + Set setOfJoinedTablesToAlsoDelete = new HashSet<>(); + if("*".equals(joinedTablesToAlsoDelete)) + { + setOfJoinedTablesToAlsoDelete.addAll(QContext.getQInstance().getTables().keySet()); + } + else if(joinedTablesToAlsoDelete != null) + { + setOfJoinedTablesToAlsoDelete.addAll(Arrays.asList(joinedTablesToAlsoDelete.split(","))); + } + + /////////////////// + // process joins // + /////////////////// + String tableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE); + lookForJoins(runBackendStepInput, tableName, runBackendStepInput.getRecords(), new HashSet<>(Set.of(tableName)), setOfJoinedTablesToAlsoDelete); + + LOG.info("GarbageCollector called with a page of records", logPair("count", runBackendStepInput.getRecords().size()), logPair("table", tableName)); + + //////////////////////////////////////////////////// + // move records (from primary table) to next step // + //////////////////////////////////////////////////// + for(QRecord qRecord : runBackendStepInput.getRecords()) + { + okSummary.incrementCountAndAddPrimaryKey(qRecord.getValue(runBackendStepInput.getTable().getPrimaryKeyField())); + runBackendStepOutput.getRecords().add(qRecord); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void postRun(BackendStepPostRunInput runBackendStepInput, BackendStepPostRunOutput runBackendStepOutput) throws QException + { + super.postRun(runBackendStepInput, runBackendStepOutput); + + /////////////////////////////////////////////////////////////////////////////////////// + // if we've just finished the validate step - // + // and if there wasn't a COUNT performed (e.g., because the table didn't support it) // + // then set our total that we accumulated into the count field. // + /////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) + { + if(runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null) + { + runBackendStepInput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, total); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void lookForJoins(RunBackendStepInput runBackendStepInput, String tableName, List records, Set visitedTables, Set allowedToAlsoDelete) throws QException + { + //////////////////////////////////////////////////////////////////////////////////////// + // if we've already visited all the tables we're allowed to delete, then return early // + //////////////////////////////////////////////////////////////////////////////////////// + HashSet anyAllowedLeft = new HashSet<>(allowedToAlsoDelete); + anyAllowedLeft.removeAll(visitedTables); + if(CollectionUtils.nullSafeIsEmpty(anyAllowedLeft)) + { + return; + } + + QInstance qInstance = QContext.getQInstance(); + JoinGraph joinGraph = qInstance.getJoinGraph(); + + //////////////////////////////////////////////////////////////////// + // get join connections from this table from the joinGraph object // + //////////////////////////////////////////////////////////////////// + Set joinConnections = joinGraph.getJoinConnections(tableName); + for(JoinGraph.JoinConnectionList joinConnectionList : CollectionUtils.nonNullCollection(joinConnections)) + { + List list = joinConnectionList.list(); + JoinGraph.JoinConnection joinConnection = list.get(0); + QJoinMetaData join = qInstance.getJoin(joinConnection.viaJoinName()); + + String recurOnTable = null; + String thisTableFKeyField = null; + String joinTablePrimaryKeyField = null; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // find the input table in the join - but only if it's on a '1' side of the join (not a many side) // + // this means we may get out of this if/else with recurOnTable = null, if we shouldn't process this join. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(join.getLeftTable().equals(tableName) && (join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.ONE_TO_ONE))) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this table is on the left side of this join, and it's a 1-n or 1-1, then delete from the right table // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + recurOnTable = join.getRightTable(); + thisTableFKeyField = join.getJoinOns().get(0).getLeftField(); + joinTablePrimaryKeyField = join.getJoinOns().get(0).getRightField(); + } + else if(join.getRightTable().equals(tableName) && (join.getType().equals(JoinType.MANY_TO_ONE) || join.getType().equals(JoinType.ONE_TO_ONE))) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else if this table is on the right side of this join, and it's n-1 or 1-1, then delete from the left table // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + recurOnTable = join.getLeftTable(); + thisTableFKeyField = join.getJoinOns().get(0).getRightField(); + joinTablePrimaryKeyField = join.getJoinOns().get(0).getLeftField(); + } + + ////////////////////////////////////////////////////////////////////////////////////////////// + // if we found a table to 'recur' on, and we haven't visited it before, then process it now // + ////////////////////////////////////////////////////////////////////////////////////////////// + if(recurOnTable != null && !visitedTables.contains(recurOnTable)) + { + if(join.getJoinOns().size() > 1) + { + LOG.warn("We would delete child records from the join [" + join.getName() + "], but it has multiple joinOns, and we don't support that yet..."); + continue; + } + + visitedTables.add(recurOnTable); + + //////////////////////////////////////////////////////////// + // query for records in the child table based on the join // + //////////////////////////////////////////////////////////// + QTableMetaData foreignTable = qInstance.getTable(recurOnTable); + String finalThisTableFKeyField = thisTableFKeyField; + List foreignKeys = records.stream().map(r -> r.getValue(finalThisTableFKeyField)).distinct().toList(); + List foreignRecords = new QueryAction().execute(new QueryInput(recurOnTable).withFilter(new QQueryFilter(new QFilterCriteria(joinTablePrimaryKeyField, IN, foreignKeys)))).getRecords(); + + //////////////////////////////////////////////////////////////////////////////////// + // make a recursive call looking for children of this table // + // we do this before we delete from this table, so that the children can be found // + //////////////////////////////////////////////////////////////////////////////////// + lookForJoins(runBackendStepInput, recurOnTable, foreignRecords, visitedTables, allowedToAlsoDelete); + + if(allowedToAlsoDelete.contains(recurOnTable)) + { + LOG.info("Deleting descendant records from: " + recurOnTable); + descendantRecordCountToDelete.putIfAbsent(foreignTable.getLabel(), 0); + descendantRecordCountToDelete.put(foreignTable.getLabel(), descendantRecordCountToDelete.get(foreignTable.getLabel()) + foreignRecords.size()); + + ///////////////////////////////////////////////////////////////////// + // if this is the execute step - then do it - delete the children. // + ///////////////////////////////////////////////////////////////////// + if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE)) + { + List foreignPKeys = foreignRecords.stream().map(r -> r.getValue(foreignTable.getPrimaryKeyField())).toList(); + new DeleteAction().execute(new DeleteInput(recurOnTable).withPrimaryKeys(foreignPKeys).withTransaction(getTransaction().orElse(null))); + } + } + } + } + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTest.java new file mode 100644 index 00000000..1ad4dfb6 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTest.java @@ -0,0 +1,328 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector; + + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for GarbageCollectorTransformStep + *******************************************************************************/ +class GarbageCollectorTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBasic() throws QException + { + QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_PERSON_MEMORY, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), null); + QContext.getQInstance().addProcess(process); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords())); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(process.getName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter())); + assertEquals(2, queryOutput.getRecords().size()); + assertEquals(Set.of(4, 5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List getPersonRecords() + { + List records = List.of( + new QRecord().withValue("id", 1).withValue("createDate", Instant.now().minus(90, ChronoUnit.DAYS)), + new QRecord().withValue("id", 2).withValue("createDate", Instant.now().minus(31, ChronoUnit.DAYS)), + new QRecord().withValue("id", 3).withValue("createDate", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)), + new QRecord().withValue("id", 4).withValue("createDate", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)), + new QRecord().withValue("id", 5).withValue("createDate", Instant.now().minus(5, ChronoUnit.DAYS))); + return records; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOverrideDate() throws QException + { + QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_PERSON_MEMORY, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), null); + QContext.getQInstance().addProcess(process); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords())); + + /////////////////////////////////////////////////////////////// + // run with a limit of 100 days ago, and 0 should be deleted // + /////////////////////////////////////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(process.getName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.addValue("limitDate", Instant.now().minus(100, ChronoUnit.DAYS)); + new RunProcessAction().execute(input); + + QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter())); + assertEquals(5, queryOutput.getRecords().size()); + + /////////////////////////////////////////////////// + // re-run with 10 days, and all but 1 be deleted // + /////////////////////////////////////////////////// + input.addValue("limitDate", Instant.now().minus(10, ChronoUnit.DAYS)); + new RunProcessAction().execute(input); + + queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter())); + assertEquals(1, queryOutput.getRecords().size()); + + /////////////////////////////////////////////// + // re-run with 1 day, and all end up deleted // + /////////////////////////////////////////////// + input.addValue("limitDate", Instant.now().minus(1, ChronoUnit.DAYS)); + new RunProcessAction().execute(input); + + queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter())); + assertEquals(0, queryOutput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithDeleteAllJoins() throws QException + { + QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), "*"); + QContext.getQInstance().addProcess(process); + + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_ORDER).withRecords(getOrderRecords())); + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM).withRecords(getLineItemRecords())); + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withRecords(getLineItemExtrinsicRecords())); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(process.getName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + QueryOutput orderQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_ORDER).withFilter(new QQueryFilter())); + assertEquals(2, orderQueryOutput.getRecords().size()); + assertEquals(Set.of(4, 5), orderQueryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet())); + + QueryOutput lineItemQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM).withFilter(new QQueryFilter())); + assertEquals(9, lineItemQueryOutput.getRecords().size()); + assertEquals(Set.of(4, 5), lineItemQueryOutput.getRecords().stream().map(r -> r.getValueInteger("orderId")).collect(Collectors.toSet())); + + QueryOutput lineItemExtrinsicQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withFilter(new QQueryFilter())); + assertEquals(5, lineItemExtrinsicQueryOutput.getRecords().size()); + assertEquals(Set.of(7, 9, 11, 13, 15), lineItemExtrinsicQueryOutput.getRecords().stream().map(r -> r.getValueInteger("lineItemId")).collect(Collectors.toSet())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithDeleteSomeJoins() throws QException + { + QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), TestUtils.TABLE_NAME_LINE_ITEM); + QContext.getQInstance().addProcess(process); + + ////////////////////////////////////////////////////////////////////////// + // remove table's associations - as they implicitly cascade the delete! // + ////////////////////////////////////////////////////////////////////////// + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER).withAssociations(new ArrayList<>()); + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_LINE_ITEM).withAssociations(new ArrayList<>()); + + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_ORDER).withRecords(getOrderRecords())); + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM).withRecords(getLineItemRecords())); + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withRecords(getLineItemExtrinsicRecords())); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(process.getName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + QueryOutput orderQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_ORDER).withFilter(new QQueryFilter())); + assertEquals(2, orderQueryOutput.getRecords().size()); + assertEquals(Set.of(4, 5), orderQueryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet())); + + QueryOutput lineItemQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM).withFilter(new QQueryFilter())); + assertEquals(9, lineItemQueryOutput.getRecords().size()); + assertEquals(Set.of(4, 5), lineItemQueryOutput.getRecords().stream().map(r -> r.getValueInteger("orderId")).collect(Collectors.toSet())); + + QueryOutput lineItemExtrinsicQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withFilter(new QQueryFilter())); + assertEquals(8, lineItemExtrinsicQueryOutput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithDeleteNoJoins() throws QException + { + QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), null); + QContext.getQInstance().addProcess(process); + + //////////////////////////////////////////////////////////////////////////////// + // remove order table's associations - as they implicitly cascade the delete! // + //////////////////////////////////////////////////////////////////////////////// + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER).withAssociations(new ArrayList<>()); + + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_ORDER).withRecords(getOrderRecords())); + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM).withRecords(getLineItemRecords())); + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withRecords(getLineItemExtrinsicRecords())); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(process.getName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + QueryOutput orderQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_ORDER).withFilter(new QQueryFilter())); + assertEquals(2, orderQueryOutput.getRecords().size()); + assertEquals(Set.of(4, 5), orderQueryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet())); + + QueryOutput lineItemQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM).withFilter(new QQueryFilter())); + assertEquals(15, lineItemQueryOutput.getRecords().size()); + + QueryOutput lineItemExtrinsicQueryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).withFilter(new QQueryFilter())); + assertEquals(8, lineItemExtrinsicQueryOutput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List getOrderRecords() + { + List records = List.of( + new QRecord().withValue("id", 1).withValue("createDate", Instant.now().minus(90, ChronoUnit.DAYS)), + new QRecord().withValue("id", 2).withValue("createDate", Instant.now().minus(31, ChronoUnit.DAYS)), + new QRecord().withValue("id", 3).withValue("createDate", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)), + new QRecord().withValue("id", 4).withValue("createDate", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)), + new QRecord().withValue("id", 5).withValue("createDate", Instant.now().minus(5, ChronoUnit.DAYS))); + return records; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List getLineItemRecords() + { + List records = List.of( + new QRecord().withValue("id", 1).withValue("orderId", 1), + new QRecord().withValue("id", 2).withValue("orderId", 2), + new QRecord().withValue("id", 3).withValue("orderId", 2), + new QRecord().withValue("id", 4).withValue("orderId", 3), + new QRecord().withValue("id", 5).withValue("orderId", 3), + new QRecord().withValue("id", 6).withValue("orderId", 3), + new QRecord().withValue("id", 7).withValue("orderId", 4), + new QRecord().withValue("id", 8).withValue("orderId", 4), + new QRecord().withValue("id", 9).withValue("orderId", 4), + new QRecord().withValue("id", 10).withValue("orderId", 4), + new QRecord().withValue("id", 11).withValue("orderId", 5), + new QRecord().withValue("id", 12).withValue("orderId", 5), + new QRecord().withValue("id", 13).withValue("orderId", 5), + new QRecord().withValue("id", 14).withValue("orderId", 5), + new QRecord().withValue("id", 15).withValue("orderId", 5)); + + return records; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List getLineItemExtrinsicRecords() + { + List records = List.of( + new QRecord().withValue("id", 1).withValue("lineItemId", 1), + new QRecord().withValue("id", 2).withValue("lineItemId", 3), + new QRecord().withValue("id", 3).withValue("lineItemId", 5), + new QRecord().withValue("id", 4).withValue("lineItemId", 7), + new QRecord().withValue("id", 5).withValue("lineItemId", 9), + new QRecord().withValue("id", 6).withValue("lineItemId", 11), + new QRecord().withValue("id", 7).withValue("lineItemId", 13), + new QRecord().withValue("id", 8).withValue("lineItemId", 15)); + + return records; + } + +} \ No newline at end of file