QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Checkpoint - working versions of streamed with frontend processes, with validation

Browse Source
This commit is contained in:
darin.kelkhoff
2022-08-29 13:33:35 -05:00
parent 5f8f063b99
commit cb22f86793
31 changed files with 1253 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java
View File

@ -22,7 +22,10 @@
package com.kingsrook.sampleapp;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
@ -49,6 +52,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
@ -56,6 +62,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFor
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep;
import io.github.cdimascio.dotenv.Dotenv;
@ -79,6 +86,7 @@ public class SampleMetaDataProvider
public static final String PROCESS_NAME_GREET = "greet";
public static final String PROCESS_NAME_GREET_INTERACTIVE = "greetInteractive";
public static final String PROCESS_NAME_CLONE_PEOPLE = "clonePeople";
public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep";
public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow";
public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive";
@ -110,6 +118,7 @@ public class SampleMetaDataProvider
qInstance.addTable(defineTableCityFile());
qInstance.addProcess(defineProcessGreetPeople());
qInstance.addProcess(defineProcessGreetPeopleInteractive());
qInstance.addProcess(defineProcessClonePeople());
qInstance.addProcess(defineProcessSimpleSleep());
qInstance.addProcess(defineProcessScreenThenSleep());
qInstance.addProcess(defineProcessSimpleThrow());
@ -129,19 +138,18 @@ public class SampleMetaDataProvider
qInstance.addApp(new QAppMetaData()
.withName(APP_NAME_GREETINGS)
.withIcon(new QIcon().withName("emoji_people"))
.withChild(qInstance.getProcess(PROCESS_NAME_GREET)
.withIcon(new QIcon().withName("emoji_people")))
.withChild(qInstance.getTable(TABLE_NAME_PERSON)
.withIcon(new QIcon().withName("person")))
.withChild(qInstance.getTable(TABLE_NAME_CITY)
.withIcon(new QIcon().withName("location_city")))
.withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE))
.withIcon(new QIcon().withName("waving_hand")));
.withChild(qInstance.getProcess(PROCESS_NAME_GREET).withIcon(new QIcon().withName("emoji_people")))
.withChild(qInstance.getTable(TABLE_NAME_PERSON).withIcon(new QIcon().withName("person")))
.withChild(qInstance.getTable(TABLE_NAME_CITY).withIcon(new QIcon().withName("location_city")))
.withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE).withIcon(new QIcon().withName("waving_hand")))
);
qInstance.addApp(new QAppMetaData()
.withName(APP_NAME_PEOPLE)
.withIcon(new QIcon().withName("person"))
.withChild(qInstance.getApp(APP_NAME_GREETINGS)));
.withChild(qInstance.getApp(APP_NAME_GREETINGS))
.withChild(qInstance.getProcess(PROCESS_NAME_CLONE_PEOPLE).withIcon(new QIcon().withName("content_copy")))
);
qInstance.addApp(new QAppMetaData()
.withName(APP_NAME_MISCELLANEOUS)
@ -231,8 +239,8 @@ public class SampleMetaDataProvider
.withBackendName("company_code"));
table.addField(new QFieldMetaData("service_level", QFieldType.STRING) // todo PVS
.withLabel("Service Level")
.withIsRequired(true));
.withLabel("Service Level")
.withIsRequired(true));
table.addSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "name")));
table.addSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("company_code", "service_level")));
@ -325,7 +333,7 @@ public class SampleMetaDataProvider
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
.withField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
);
@ -365,7 +373,7 @@ public class SampleMetaDataProvider
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
.withField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
)
@ -383,6 +391,35 @@ public class SampleMetaDataProvider
/*******************************************************************************
**
*******************************************************************************/
private static QProcessMetaData defineProcessClonePeople()
{
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, TABLE_NAME_PERSON);
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, TABLE_NAME_PERSON);
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
ExtractViaQueryStep.class,
ClonePeopleTransformStep.class,
LoadViaInsertStep.class,
values
);
process.setName(PROCESS_NAME_CLONE_PEOPLE);
process.setTableName(TABLE_NAME_PERSON);
process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)
.withRecordListField(new QFieldMetaData("firstName", QFieldType.STRING))
.withRecordListField(new QFieldMetaData("lastName", QFieldType.STRING))
;
return (process);
}
/*******************************************************************************
** Define a process with just one step that sleeps
*******************************************************************************/
@ -467,7 +504,7 @@ public class SampleMetaDataProvider
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(SleeperStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER))));
.withField(new QFieldMetaData(SleeperStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER))));
}
}

88
qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java Normal file
View File

@ -0,0 +1,88 @@
package com.kingsrook.sampleapp.processes.clonepeople;
import java.io.Serializable;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
/*******************************************************************************
**
*******************************************************************************/
public class ClonePeopleTransformStep extends AbstractTransformStep implements ProcessSummaryProviderInterface
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK, 0, "can be cloned with no issues.");
private ProcessSummaryLine warningCloneSummary = new ProcessSummaryLine(Status.WARNING, 0, "can be cloned, but because are already a clone, their clone cannot be cloned in the future.");
private ProcessSummaryLine refuseCloningSummary = new ProcessSummaryLine(Status.ERROR, 0, "say they don't want to be cloned (probably a Garret...)");
private ProcessSummaryLine nestedCloneSummary = new ProcessSummaryLine(Status.ERROR, 0, "are already a clone of a clone, so they can't be cloned again.");
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
{
if(isForResultScreen)
{
okSummary.setMessage("were cloned");
warningCloneSummary.setMessage("were already a clone, so they were cloned again now, but their clones cannot be cloned after this.");
nestedCloneSummary.setMessage("are already a clone of a clone, so they weren't cloned again.");
}
ArrayList<ProcessSummaryLine> rs = new ArrayList<>();
okSummary.addSelfToListIfAnyCount(rs);
warningCloneSummary.addSelfToListIfAnyCount(rs);
refuseCloningSummary.addSelfToListIfAnyCount(rs);
nestedCloneSummary.addSelfToListIfAnyCount(rs);
return (rs);
}
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
for(QRecord inputPerson : getInputRecordPage())
{
Serializable id = inputPerson.getValue("id");
if("Garret".equals(inputPerson.getValueString("firstName")))
{
refuseCloningSummary.incrementCountAndAddPrimaryKey(id);
}
else if(inputPerson.getValueString("firstName").matches("Clone of.*Clone of.*"))
{
nestedCloneSummary.incrementCountAndAddPrimaryKey(id);
}
else
{
QRecord outputPerson = new QRecord(inputPerson);
outputPerson.setValue("id", null);
outputPerson.setValue("firstName", "Clone of: " + inputPerson.getValueString("firstName"));
getOutputRecordPage().add(outputPerson);
if(inputPerson.getValueString("firstName").matches("Clone of.*"))
{
warningCloneSummary.incrementCountAndAddPrimaryKey(id);
}
else
{
okSummary.incrementCountAndAddPrimaryKey(id);
}
}
}
}
}