Add TableSyncProcess

This commit is contained in:
2022-11-11 12:28:44 -06:00
parent a5ec33b51b
commit 8b31cee890
20 changed files with 1138 additions and 29 deletions

View File

@ -189,6 +189,7 @@ class ExportActionTest
{
QTableMetaData wideTable = new QTableMetaData()
.withName("wide")
.withPrimaryKeyField("field0")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME);
for(int i = 0; i < ReportFormat.XLSX.getMaxCols() + 1; i++)
{

View File

@ -129,7 +129,6 @@ public class QPossibleValueTranslatorTest
{
QInstance qInstance = TestUtils.defineInstance();
QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
QTableMetaData shapeTable = qInstance.getTable(TestUtils.TABLE_NAME_SHAPE);
QFieldMetaData shapeField = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId");
QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(shapeField.getPossibleValueSourceName());
@ -195,6 +194,32 @@ public class QPossibleValueTranslatorTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testPossibleValueTableWithBadForeignKeys() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
QFieldMetaData shapeField = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId");
TestUtils.insertDefaultShapes(qInstance);
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// assert that we don't re-run queries for cached values, even ones that aren't found (e.g., 4 below). //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
MemoryRecordStore.setCollectStatistics(true);
possibleValueTranslator.translatePossibleValue(shapeField, 1);
possibleValueTranslator.translatePossibleValue(shapeField, 2);
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 queries so far");
assertNull(possibleValueTranslator.translatePossibleValue(shapeField, 4));
assertNull(possibleValueTranslator.translatePossibleValue(shapeField, 4));
assertEquals(3, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 3 queries in total");
}
/*******************************************************************************
** Make sure that if we have 2 different PVS's pointed at the same 1 table,
** that we avoid re-doing queries, and that we actually get different (formatted) values.

View File

@ -354,10 +354,10 @@ class QInstanceValidatorTest
public void test_validateTableWithNoFields()
{
assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(null),
"At least 1 field");
"At least 1 field", "Primary key for table person is not a recognized field");
assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(new HashMap<>()),
"At least 1 field");
"At least 1 field", "Primary key for table person is not a recognized field");
}
@ -547,6 +547,7 @@ class QInstanceValidatorTest
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection(null, "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "Missing a name");
@ -562,6 +563,7 @@ class QInstanceValidatorTest
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", null, new QIcon("person"), Tier.T1, List.of("id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationSuccess((qInstance) -> qInstance.addTable(table));
@ -577,6 +579,7 @@ class QInstanceValidatorTest
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withSection(new QFieldSection("section1", "Section 2", new QIcon("person"), Tier.T2, List.of("name")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
@ -594,6 +597,7 @@ class QInstanceValidatorTest
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withSection(new QFieldSection("section2", "Section 1", new QIcon("person"), Tier.T2, List.of("name")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
@ -611,12 +615,14 @@ class QInstanceValidatorTest
{
QTableMetaData table1 = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of()))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "section1 does not have any fields", "field id is not listed in any field sections");
QTableMetaData table2 = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, null))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table2), "section1 does not have any fields", "field id is not listed in any field sections");
@ -632,6 +638,7 @@ class QInstanceValidatorTest
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "od")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "not a field on this table");
@ -647,12 +654,14 @@ class QInstanceValidatorTest
{
QTableMetaData table1 = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "more than once");
QTableMetaData table2 = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T2, List.of("id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
@ -669,6 +678,7 @@ class QInstanceValidatorTest
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("name", QFieldType.STRING));
@ -685,6 +695,7 @@ class QInstanceValidatorTest
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T1, List.of("name")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
@ -800,6 +811,7 @@ class QInstanceValidatorTest
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("name", QFieldType.STRING));

View File

@ -0,0 +1,184 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecordEnum;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for EnumerationCountAction
*******************************************************************************/
class EnumerationCountActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUnfilteredCount() throws QException
{
QInstance instance = defineQInstance();
CountInput countInput = new CountInput(instance);
countInput.setSession(new QSession());
countInput.setTableName("statesEnum");
CountOutput countOutput = new CountAction().execute(countInput);
assertEquals(2, countOutput.getCount());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFilteredCount() throws QException
{
QInstance instance = defineQInstance();
CountInput countInput = new CountInput(instance);
countInput.setSession(new QSession());
countInput.setTableName("statesEnum");
countInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("population", QCriteriaOperator.GREATER_THAN, List.of(20_000_000))));
CountOutput countOutput = new CountAction().execute(countInput);
assertEquals(1, countOutput.getCount());
}
/*******************************************************************************
**
*******************************************************************************/
private QInstance defineQInstance()
{
QInstance instance = TestUtils.defineInstance();
instance.addBackend(new QBackendMetaData()
.withName("enum")
.withBackendType("enum")
);
instance.addTable(new QTableMetaData()
.withName("statesEnum")
.withBackendName("enum")
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("name", QFieldType.STRING))
.withField(new QFieldMetaData("postalCode", QFieldType.STRING))
.withField(new QFieldMetaData("population", QFieldType.INTEGER))
.withBackendDetails(new EnumerationTableBackendDetails().withEnumClass(States.class))
);
return instance;
}
/*******************************************************************************
**
*******************************************************************************/
public static enum States implements QRecordEnum
{
MO(1, "Missouri", "MO", 15_000_000),
IL(2, "Illinois", "IL", 25_000_000);
private final Integer id;
private final String name;
private final String postalCode;
private final Integer population;
/*******************************************************************************
**
*******************************************************************************/
States(int id, String name, String postalCode, int population)
{
this.id = id;
this.name = name;
this.postalCode = postalCode;
this.population = population;
}
/*******************************************************************************
** Getter for id
**
*******************************************************************************/
public Integer getId()
{
return id;
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Getter for postalCode
**
*******************************************************************************/
public String getPostalCode()
{
return postalCode;
}
/*******************************************************************************
** Getter for population
**
*******************************************************************************/
public Integer getPopulation()
{
return population;
}
}
}

View File

@ -0,0 +1,147 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for TableSyncProcess
*******************************************************************************/
class TableSyncProcessTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData personTable = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
String TABLE_NAME_PEOPLE_SYNC = "peopleSync";
qInstance.addTable(new QTableMetaData()
.withName(TABLE_NAME_PEOPLE_SYNC)
.withPrimaryKeyField("id")
.withBackendName(TestUtils.MEMORY_BACKEND_NAME)
.withFields(personTable.getFields())
.withField(new QFieldMetaData("sourcePersonId", QFieldType.INTEGER)));
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of(
new QRecord().withValue("id", 1).withValue("firstName", "Darin"),
new QRecord().withValue("id", 2).withValue("firstName", "Tim"),
new QRecord().withValue("id", 3).withValue("firstName", "Tyler"),
new QRecord().withValue("id", 4).withValue("firstName", "James"),
new QRecord().withValue("id", 5).withValue("firstName", "Homer")
));
TestUtils.insertRecords(qInstance, qInstance.getTable(TABLE_NAME_PEOPLE_SYNC), List.of(
new QRecord().withValue("sourcePersonId", 3).withValue("firstName", "Garret"),
new QRecord().withValue("sourcePersonId", 5).withValue("firstName", "Homer")
));
String PROCESS_NAME = "testSyncProcess";
qInstance.addProcess(TableSyncProcess.processMetaDataBuilder(false)
.withName(PROCESS_NAME)
.withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withDestinationTable(TABLE_NAME_PEOPLE_SYNC)
.withSourceTableKeyField("id")
.withDestinationTableForeignKeyField("sourcePersonId")
.withSyncTransformStepClass(PersonTransformClass.class)
.getProcessMetaData());
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(new QSession());
runProcessInput.setProcessName(PROCESS_NAME);
runProcessInput.addValue("recordIds", "1,2,3,4,5");
runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
RunProcessAction runProcessAction = new RunProcessAction();
RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput);
@SuppressWarnings("unchecked")
ArrayList<ProcessSummaryLineInterface> processResults = (ArrayList<ProcessSummaryLineInterface>) runProcessOutput.getValues().get("processResults");
assertThat(processResults.get(0))
.hasFieldOrPropertyWithValue("message", "were inserted")
.hasFieldOrPropertyWithValue("count", 3);
assertThat(processResults.get(1))
.hasFieldOrPropertyWithValue("message", "were updated")
.hasFieldOrPropertyWithValue("count", 2);
List<QRecord> syncedRecords = TestUtils.queryTable(qInstance, TABLE_NAME_PEOPLE_SYNC);
assertEquals(5, syncedRecords.size());
/////////////////////////////////////////////////////////////////
// make sure the record referencing 3 has had its name updated //
// and the one referencing 5 stayed the same //
/////////////////////////////////////////////////////////////////
Map<Serializable, QRecord> syncPersonsBySourceId = GeneralProcessUtils.loadTableToMap(runProcessInput, TABLE_NAME_PEOPLE_SYNC, "sourcePersonId");
assertEquals("Tyler", syncPersonsBySourceId.get(3).getValueString("firstName"));
assertEquals("Homer", syncPersonsBySourceId.get(5).getValueString("firstName"));
}
/*******************************************************************************
**
*******************************************************************************/
public static class PersonTransformClass extends AbstractTableSyncTransformStep
{
@Override
public QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException
{
destinationRecord.setValue("sourcePersonId", sourceRecord.getValue("id"));
destinationRecord.setValue("firstName", sourceRecord.getValue("firstName"));
destinationRecord.setValue("lastName", sourceRecord.getValue("lastName"));
return (destinationRecord);
}
}
}