QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4 from Kingsrook/feature/sprint-8

Browse Source 
Feature/sprint 8
This commit is contained in:
darin.kelkhoff
2022-08-11 09:49:03 -05:00
committed by GitHub
parent 509b6f783b 3b1f0b47c1
commit 7e2af98e01
95 changed files with 5502 additions and 520 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.circleci/config.yml
View File

@ -31,10 +31,14 @@ commands:
- restore_cache:
keys:
- v1-dependencies-{{ checksum "pom.xml" }}
- run:
name: Write .env
command: |
echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> qqq-sample-project/.env
- run:
name: Run Maven Verify
command: |
mvn -s .circleci/mvn-settings.xml verify
mvn -s .circleci/mvn-settings.xml -T4 verify
- store_jacoco_site:
module: qqq-backend-core
- store_jacoco_site:
@ -69,7 +73,7 @@ commands:
- run:
name: Run Maven Jar Deploy
command: |
mvn -s .circleci/mvn-settings.xml jar:jar deploy:deploy
mvn -s .circleci/mvn-settings.xml -T4 flatten:flatten jar:jar deploy:deploy
- save_cache:
paths:
- ~/.m2
@ -99,7 +103,7 @@ workflows:
test_only:
jobs:
- mvn_test:
context: [ qqq-maven-registry-credentials, kingsrook-slack ]
context: [ qqq-maven-registry-credentials, kingsrook-slack, build-qqq-sample-app ]
filters:
branches:
ignore: /dev/
@ -109,7 +113,7 @@ workflows:
deploy:
jobs:
- mvn_deploy:
context: [ qqq-maven-registry-credentials, kingsrook-slack ]
context: [ qqq-maven-registry-credentials, kingsrook-slack, build-qqq-sample-app ]
filters:
branches:
only: /dev/

1
.gitignore vendored
View File

@ -31,3 +31,4 @@ target/
hs_err_pid*
.DS_Store
*.swp
.flattened-pom.xml

25
pom.xml
View File

@ -137,6 +137,31 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>resolveCiFriendliesOnly</flattenMode>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.amashchenko.maven.plugin</groupId>
<artifactId>gitflow-maven-plugin</artifactId>

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java
View File

@ -30,7 +30,7 @@ import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModu
/*******************************************************************************
**
** Utility methods to be shared by all of the various Actions (e.g., InsertAction)
*******************************************************************************/
public class ActionHelper
{
@ -42,7 +42,7 @@ public class ActionHelper
{
QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher();
QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(request.getAuthenticationMetaData());
if(!authenticationModule.isSessionValid(request.getSession()))
if(!authenticationModule.isSessionValid(request.getInstance(), request.getSession()))
{
throw new QAuthenticationException("Invalid session in request");
}

73
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java Normal file
View File

@ -0,0 +1,73 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
/*******************************************************************************
** Container wherein backend modules can track data and/or objects that are
** part of a transaction.
**
** Most obvious use-case would be a JDBC Connection. See subclass in rdbms module.
**
** Note: One would imagine that this class shouldn't ever implement Serializable...
*******************************************************************************/
public class QBackendTransaction
{
/*******************************************************************************
** Commit the transaction.
*******************************************************************************/
public void commit() throws QException
{
////////////////////////
// noop in base class //
////////////////////////
}
/*******************************************************************************
** Rollback the transaction.
*******************************************************************************/
public void rollback() throws QException
{
////////////////////////
// noop in base class //
////////////////////////
}
/*******************************************************************************
** Close any resources associated with the transaction. In theory, should only
** be called after a commit or rollback was done.
*******************************************************************************/
public void close()
{
////////////////////////
// noop in base class //
////////////////////////
}
}

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.actions.interfaces;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@ -37,4 +38,13 @@ public interface InsertInterface
**
*******************************************************************************/
InsertOutput execute(InsertInput insertInput) throws QException;
/*******************************************************************************
**
*******************************************************************************/
default QBackendTransaction openTransaction(InsertInput insertInput) throws QException
{
return (new QBackendTransaction());
}
}

73
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java
View File

@ -22,14 +22,20 @@
package com.kingsrook.qqq.backend.core.actions.metadata;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -50,22 +56,87 @@ public class MetaDataAction
// todo pre-customization - just get to modify the request?
MetaDataOutput metaDataOutput = new MetaDataOutput();
Map<String, AppTreeNode> treeNodes = new LinkedHashMap<>();
/////////////////////////////////////
// map tables to frontend metadata //
/////////////////////////////////////
Map<String, QFrontendTableMetaData> tables = new LinkedHashMap<>();
for(Map.Entry<String, QTableMetaData> entry : metaDataInput.getInstance().getTables().entrySet())
{
tables.put(entry.getKey(), new QFrontendTableMetaData(entry.getValue(), false));
treeNodes.put(entry.getKey(), new AppTreeNode(entry.getValue()));
}
metaDataOutput.setTables(tables);
////////////////////////////////////////
// map processes to frontend metadata //
////////////////////////////////////////
Map<String, QFrontendProcessMetaData> processes = new LinkedHashMap<>();
for(Map.Entry<String, QProcessMetaData> entry : metaDataInput.getInstance().getProcesses().entrySet())
{
processes.put(entry.getKey(), new QFrontendProcessMetaData(entry.getValue(), false));
treeNodes.put(entry.getKey(), new AppTreeNode(entry.getValue()));
}
metaDataOutput.setProcesses(processes);
// todo post-customization - can do whatever w/ the result if you want
///////////////////////////////////
// map apps to frontend metadata //
///////////////////////////////////
Map<String, QFrontendAppMetaData> apps = new LinkedHashMap<>();
for(Map.Entry<String, QAppMetaData> entry : metaDataInput.getInstance().getApps().entrySet())
{
apps.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue()));
treeNodes.put(entry.getKey(), new AppTreeNode(entry.getValue()));
for(QAppChildMetaData child : entry.getValue().getChildren())
{
apps.get(entry.getKey()).addChild(new AppTreeNode(child));
}
}
metaDataOutput.setApps(apps);
////////////////////////////////////////////////
// organize app tree nodes by their hierarchy //
////////////////////////////////////////////////
List<AppTreeNode> appTree = new ArrayList<>();
for(QAppMetaData appMetaData : metaDataInput.getInstance().getApps().values())
{
if(appMetaData.getParentAppName() == null)
{
buildAppTree(treeNodes, appTree, appMetaData);
}
}
metaDataOutput.setAppTree(appTree);
// todo post-customization - can do whatever w/ the result if you want?
return metaDataOutput;
}
/*******************************************************************************
**
*******************************************************************************/
private void buildAppTree(Map<String, AppTreeNode> treeNodes, List<AppTreeNode> nodeList, QAppChildMetaData childMetaData)
{
AppTreeNode treeNode = treeNodes.get(childMetaData.getName());
if(treeNode == null)
{
return;
}
nodeList.add(treeNode);
if(childMetaData instanceof QAppMetaData app)
{
if(app.getChildren() != null)
{
for(QAppChildMetaData child : app.getChildren())
{
buildAppTree(treeNodes, treeNode.getChildren(), child);
}
}
}
}
}

21
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java
View File

@ -25,7 +25,11 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -34,16 +38,25 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
*******************************************************************************/
public class RecordPipe
{
private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(10_000);
private static final Logger LOG = LogManager.getLogger(RecordPipe.class);
private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(1_000);
/*******************************************************************************
** Add a record to the pipe
** Returns true iff the record fit in the pipe; false if the pipe is currently full.
*******************************************************************************/
public boolean addRecord(QRecord record)
public void addRecord(QRecord record)
{
return (queue.offer(record));
boolean offerResult = queue.offer(record);
while(!offerResult)
{
LOG.debug("Record pipe.add failed (due to full pipe). Blocking.");
SleepUtils.sleep(100, TimeUnit.MILLISECONDS);
offerResult = queue.offer(record);
}
}
@ -53,7 +66,7 @@ public class RecordPipe
*******************************************************************************/
public void addRecords(List<QRecord> records)
{
queue.addAll(records);
records.forEach(this::addRecord);
}

30
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@ -47,14 +48,35 @@ public class InsertAction
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend());
QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
// todo pre-customization - just get to modify the request?
InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput);
// todo post-customization - can do whatever w/ the result if you want
return insertOutput;
}
/*******************************************************************************
**
*******************************************************************************/
private QBackendModuleInterface getBackendModuleInterface(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend());
return (qModule);
}
/*******************************************************************************
**
*******************************************************************************/
public QBackendTransaction openTransaction(InsertInput insertInput) throws QException
{
QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
return (qModule.getInsertInterface().openTransaction(insertInput));
}
}

8
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
@ -48,6 +49,13 @@ public class QueryAction
// todo pre-customization - just get to modify the request?
QueryOutput queryOutput = qModule.getQueryInterface().execute(queryInput);
// todo post-customization - can do whatever w/ the result if you want
if (queryInput.getRecordPipe() == null)
{
QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), queryOutput.getRecords());
}
return queryOutput;
}
}

178
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java Normal file
View File

@ -0,0 +1,178 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.values;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Utility to apply display formats to values for fields
*******************************************************************************/
public class QValueFormatter
{
private static final Logger LOG = LogManager.getLogger(QValueFormatter.class);
/*******************************************************************************
**
*******************************************************************************/
public static String formatValue(QFieldMetaData field, Serializable value)
{
//////////////////////////////////
// null values get null results //
//////////////////////////////////
if(value == null)
{
return (null);
}
////////////////////////////////////////////////////////
// if the field has a display format, try to apply it //
////////////////////////////////////////////////////////
if(StringUtils.hasContent(field.getDisplayFormat()))
{
try
{
return (field.getDisplayFormat().formatted(value));
}
catch(Exception e)
{
try
{
if(e.getMessage().equals("f != java.lang.Integer"))
{
return formatValue(field, ValueUtils.getValueAsBigDecimal(value));
}
else if(e.getMessage().equals("d != java.math.BigDecimal"))
{
return formatValue(field, ValueUtils.getValueAsInteger(value));
}
else
{
LOG.warn("Error formatting value [" + value + "] for field [" + field.getName() + "] with format [" + field.getDisplayFormat() + "]: " + e.getMessage());
}
}
catch(Exception e2)
{
LOG.warn("Caught secondary exception trying to convert type on field [" + field.getName() + "] for formatting", e);
}
}
}
////////////////////////////////////////
// by default, just get back a string //
////////////////////////////////////////
return (ValueUtils.getValueAsString(value));
}
/*******************************************************************************
** Make a string from a table's recordLabelFormat and fields, for a given record.
*******************************************************************************/
public static String formatRecordLabel(QTableMetaData table, QRecord record)
{
if(!StringUtils.hasContent(table.getRecordLabelFormat()))
{
return (formatRecordLabelExceptionalCases(table, record));
}
///////////////////////////////////////////////////////////////////////
// get list of values, then pass them to the string formatter method //
///////////////////////////////////////////////////////////////////////
try
{
List<Serializable> values = table.getRecordLabelFields().stream()
.map(record::getValue)
.map(v -> v == null ? "" : v)
.toList();
return (table.getRecordLabelFormat().formatted(values.toArray()));
}
catch(Exception e)
{
return (formatRecordLabelExceptionalCases(table, record));
}
}
/*******************************************************************************
** Deal with non-happy-path cases for making a record label.
*******************************************************************************/
private static String formatRecordLabelExceptionalCases(QTableMetaData table, QRecord record)
{
///////////////////////////////////////////////////////////////////////////////////////
// if there's no record label format, then just return the primary key display value //
///////////////////////////////////////////////////////////////////////////////////////
String pkeyDisplayValue = record.getDisplayValue(table.getPrimaryKeyField());
if(StringUtils.hasContent(pkeyDisplayValue))
{
return (pkeyDisplayValue);
}
String pkeyRawValue = ValueUtils.getValueAsString(record.getValue(table.getPrimaryKeyField()));
if(StringUtils.hasContent(pkeyRawValue))
{
return (pkeyRawValue);
}
///////////////////////////////////////////////////////////////////////////////
// worst case scenario, return empty string, but never null from this method //
///////////////////////////////////////////////////////////////////////////////
return ("");
}
/*******************************************************************************
** For a list of records, set their recordLabels and display values
*******************************************************************************/
public static void setDisplayValuesInRecords(QTableMetaData table, List<QRecord> records)
{
if(records == null)
{
return;
}
for(QRecord record : records)
{
for(QFieldMetaData field : table.getFields().values())
{
String formattedValue = QValueFormatter.formatValue(field, record.getValue(field.getName()));
record.setDisplayValue(field.getName(), formattedValue);
}
record.setRecordLabel(QValueFormatter.formatRecordLabel(table, record));
}
}
}

83
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java
View File

@ -28,6 +28,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@ -41,9 +43,44 @@ import org.apache.commons.csv.CSVRecord;
/*******************************************************************************
** Adapter class to convert a CSV string into a list of QRecords.
**
** Based on which method is called, can either take a pipe, and stream records
** into it - or return a list of all records from the file. Either way, at this
** time, the full CSV string is read & parsed - a future optimization might read
** the CSV content from a stream as well.
*******************************************************************************/
public class CsvToQRecordAdapter
{
private RecordPipe recordPipe = null;
private List<QRecord> recordList = null;
/*******************************************************************************
** stream records from a CSV String into a RecordPipe, for a given table, optionally
** using a given mapping.
**
*******************************************************************************/
public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
{
this.recordPipe = recordPipe;
doBuildRecordsFromCsv(csv, table, mapping, recordCustomizer);
}
/*******************************************************************************
** convert a CSV String into a List of QRecords, for a given table, optionally
** using a given mapping.
**
*******************************************************************************/
public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
{
this.recordList = new ArrayList<>();
doBuildRecordsFromCsv(csv, table, mapping, null);
return (recordList);
}
/*******************************************************************************
** convert a CSV String into a List of QRecords, for a given table, optionally
@ -51,14 +88,13 @@ public class CsvToQRecordAdapter
**
** todo - meta-data validation, type handling
*******************************************************************************/
public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
public void doBuildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
{
if(!StringUtils.hasContent(csv))
{
throw (new IllegalArgumentException("Empty csv value was provided."));
}
List<QRecord> rs = new ArrayList<>();
try
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -82,7 +118,7 @@ public class CsvToQRecordAdapter
// put values from the CSV record into a map of header -> value //
//////////////////////////////////////////////////////////////////
Map<String, String> csvValues = new HashMap<>();
for(int i=0; i<headers.size(); i++)
for(int i = 0; i < headers.size(); i++)
{
csvValues.put(headers.get(i), csvRecord.get(i));
}
@ -91,12 +127,14 @@ public class CsvToQRecordAdapter
// now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
QRecord qRecord = new QRecord();
rs.add(qRecord);
for(QFieldMetaData field : table.getFields().values())
{
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
qRecord.setValue(field.getName(), csvValues.get(fieldSource));
}
runRecordCustomizer(recordCustomizer, qRecord);
addRecord(qRecord);
}
}
else if(AbstractQFieldMapping.SourceType.INDEX.equals(mapping.getSourceType()))
@ -125,12 +163,14 @@ public class CsvToQRecordAdapter
// now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
QRecord qRecord = new QRecord();
rs.add(qRecord);
for(QFieldMetaData field : table.getFields().values())
{
Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
qRecord.setValue(field.getName(), csvValues.get(fieldIndex));
}
runRecordCustomizer(recordCustomizer, qRecord);
addRecord(qRecord);
}
}
else
@ -142,8 +182,19 @@ public class CsvToQRecordAdapter
{
throw (new IllegalArgumentException("Error parsing CSV: " + e.getMessage(), e));
}
}
return (rs);
/*******************************************************************************
**
*******************************************************************************/
private void runRecordCustomizer(Consumer<QRecord> recordCustomizer, QRecord qRecord)
{
if(recordCustomizer != null)
{
recordCustomizer.accept(qRecord);
}
}
@ -165,7 +216,7 @@ public class CsvToQRecordAdapter
for(String header : headers)
{
String headerToUse = header;
String headerToUse = header;
String headerWithoutSuffix = header.replaceFirst(" \\d+$", "");
if(countsByHeader.containsKey(headerWithoutSuffix))
@ -183,4 +234,22 @@ public class CsvToQRecordAdapter
return (rs);
}
/*******************************************************************************
** Add a record - either to the pipe, or list, whichever we're building.
*******************************************************************************/
private void addRecord(QRecord record)
{
if(recordPipe != null)
{
recordPipe.addRecord(record);
}
if(recordList != null)
{
recordList.add(record);
}
}
}

215
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -23,14 +23,18 @@ package com.kingsrook.qqq.backend.core.instances;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
@ -40,14 +44,19 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMe
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteStoreStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditReceiveValuesStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditStoreRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertStoreRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -57,6 +66,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
*******************************************************************************/
public class QInstanceEnricher
{
private static final Logger LOG = LogManager.getLogger(QInstanceEnricher.class);
/*******************************************************************************
**
*******************************************************************************/
@ -77,6 +90,11 @@ public class QInstanceEnricher
{
qInstance.getBackends().values().forEach(this::enrich);
}
if(qInstance.getApps() != null)
{
qInstance.getApps().values().forEach(this::enrich);
}
}
@ -105,6 +123,11 @@ public class QInstanceEnricher
{
table.getFields().values().forEach(this::enrich);
}
if(CollectionUtils.nullSafeIsEmpty(table.getSections()))
{
generateTableFieldSections(table);
}
}
@ -172,14 +195,32 @@ public class QInstanceEnricher
/*******************************************************************************
**
*******************************************************************************/
private void enrich(QAppMetaData app)
{
if(!StringUtils.hasContent(app.getLabel()))
{
app.setLabel(nameToLabel(app.getName()));
}
}
/*******************************************************************************
**
*******************************************************************************/
private String nameToLabel(String name)
{
if(name == null)
if(!StringUtils.hasContent(name))
{
return (null);
return (name);
}
if(name.length() == 1)
{
return (name.substring(0, 1).toUpperCase(Locale.ROOT));
}
return (name.substring(0, 1).toUpperCase(Locale.ROOT) + name.substring(1).replaceAll("([A-Z])", " $1"));
@ -403,4 +444,174 @@ public class QInstanceEnricher
)));
}
/*******************************************************************************
** for all fields in a table, set their backendName, using the default "inference" logic
** see {@link #inferBackendName(String)}
*******************************************************************************/
public static void setInferredFieldBackendNames(QTableMetaData tableMetaData)
{
if(tableMetaData == null)
{
LOG.warn("Requested to infer field backend names with a null table as input. Returning with noop.");
return;
}
if(CollectionUtils.nullSafeIsEmpty(tableMetaData.getFields()))
{
LOG.warn("Requested to infer field backend names on a table [" + tableMetaData.getName() + "] with no fields. Returning with noop.");
return;
}
for(QFieldMetaData field : tableMetaData.getFields().values())
{
String fieldName = field.getName();
String fieldBackendName = field.getBackendName();
if(!StringUtils.hasContent(fieldBackendName))
{
String backendName = inferBackendName(fieldName);
field.setBackendName(backendName);
}
}
}
/*******************************************************************************
** Do a default mapping from a camelCase field name to an underscore_style
** name for a backend.
**
** Examples:
** <ul>
** <li>wordAnotherWordMoreWords -> word_another_word_more_words</li>
** <li>lUlUlUl -> l_ul_ul_ul</li>
** <li>StartsUpper -> starts_upper</li>
** <li>TLAFirst -> tla_first</li>
** <li>wordThenTLAInMiddle -> word_then_tla_in_middle</li>
** <li>endWithTLA -> end_with_tla</li>
** <li>TLAAndAnotherTLA -> tla_and_another_tla</li>
** </ul>
*******************************************************************************/
static String inferBackendName(String fieldName)
{
////////////////////////////////////////////////////////////////////////////////////////
// build a list of words in the name, then join them with _ and lower-case the result //
////////////////////////////////////////////////////////////////////////////////////////
List<String> words = new ArrayList<>();
StringBuilder currentWord = new StringBuilder();
for(int i = 0; i < fieldName.length(); i++)
{
Character thisChar = fieldName.charAt(i);
Character nextChar = i < (fieldName.length() - 1) ? fieldName.charAt(i + 1) : null;
/////////////////////////////////////////////////////////////////////////////////////
// if we're at the end of the whole string, then we're at the end of the last word //
/////////////////////////////////////////////////////////////////////////////////////
if(nextChar == null)
{
currentWord.append(thisChar);
words.add(currentWord.toString());
}
///////////////////////////////////////////////////////////
// transitioning from a lower to an upper starts a word. //
///////////////////////////////////////////////////////////
else if(Character.isLowerCase(thisChar) && Character.isUpperCase(nextChar))
{
currentWord.append(thisChar);
words.add(currentWord.toString());
currentWord = new StringBuilder();
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// transitioning from an upper to a lower - it starts a word, as long as there were already letters in the current word //
// e.g., on wordThenTLAInMiddle, when thisChar=I and nextChar=n. currentWord will be "TLA". So finish that word, and start a new one with the 'I' //
// but the normal single-upper condition, e.g., firstName, when thisChar=N and nextChar=a, current word will be empty string, so just append the 'a' to it //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
else if(Character.isUpperCase(thisChar) && Character.isLowerCase(nextChar) && currentWord.length() > 0)
{
words.add(currentWord.toString());
currentWord = new StringBuilder();
currentWord.append(thisChar);
}
/////////////////////////////////////////////////////////////
// by default, just add this character to the current word //
/////////////////////////////////////////////////////////////
else
{
currentWord.append(thisChar);
}
}
return (String.join("_", words).toLowerCase(Locale.ROOT));
}
/*******************************************************************************
** If a table didn't have any sections, generate "sensible defaults"
*******************************************************************************/
private void generateTableFieldSections(QTableMetaData table)
{
if(CollectionUtils.nullSafeIsEmpty(table.getFields()))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////
// assume this table is invalid if it has no fields, but surely it doesn't need any sections then. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
return;
}
//////////////////////////////////////////////////////////////////////////////
// create an identity section for the id and any fields in the record label //
//////////////////////////////////////////////////////////////////////////////
QFieldSection identitySection = new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, new ArrayList<>());
Set<String> usedFieldNames = new HashSet<>();
if(StringUtils.hasContent(table.getPrimaryKeyField()))
{
identitySection.getFieldNames().add(table.getPrimaryKeyField());
usedFieldNames.add(table.getPrimaryKeyField());
}
if(CollectionUtils.nullSafeHasContents(table.getRecordLabelFields()))
{
for(String fieldName : table.getRecordLabelFields())
{
if(!usedFieldNames.contains(fieldName))
{
identitySection.getFieldNames().add(fieldName);
usedFieldNames.add(fieldName);
}
}
}
if(!identitySection.getFieldNames().isEmpty())
{
table.addSection(identitySection);
}
///////////////////////////////////////////////////////////////////////////////
// if there are more fields, then add them in a default/Other Fields section //
///////////////////////////////////////////////////////////////////////////////
QFieldSection otherSection = new QFieldSection("otherFields", "Other Fields", new QIcon("dataset"), Tier.T2, new ArrayList<>());
if(CollectionUtils.nullSafeHasContents(table.getFields()))
{
for(String fieldName : table.getFields().keySet())
{
if(!usedFieldNames.contains(fieldName))
{
otherSection.getFieldNames().add(fieldName);
usedFieldNames.add(fieldName);
}
}
}
if(!otherSection.getFieldNames().isEmpty())
{
table.addSection(otherSection);
}
}
}

292
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -23,10 +23,18 @@ package com.kingsrook.qqq.backend.core.instances;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -76,60 +84,10 @@ public class QInstanceValidator
List<String> errors = new ArrayList<>();
try
{
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getBackends()),
"At least 1 backend must be defined."))
{
qInstance.getBackends().forEach((backendName, backend) ->
{
assertCondition(errors, Objects.equals(backendName, backend.getName()),
"Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
});
}
/////////////////////////
// validate the tables //
/////////////////////////
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getTables()),
"At least 1 table must be defined."))
{
qInstance.getTables().forEach((tableName, table) ->
{
assertCondition(errors, Objects.equals(tableName, table.getName()),
"Inconsistent naming for table: " + tableName + "/" + table.getName() + ".");
////////////////////////////////////////
// validate the backend for the table //
////////////////////////////////////////
if(assertCondition(errors, StringUtils.hasContent(table.getBackendName()),
"Missing backend name for table " + tableName + "."))
{
if(CollectionUtils.nullSafeHasContents(qInstance.getBackends()))
{
assertCondition(errors, qInstance.getBackendForTable(tableName) != null,
"Unrecognized backend " + table.getBackendName() + " for table " + tableName + ".");
}
}
//////////////////////////////////
// validate fields in the table //
//////////////////////////////////
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(table.getFields()),
"At least 1 field must be defined in table " + tableName + "."))
{
table.getFields().forEach((fieldName, field) ->
{
assertCondition(errors, Objects.equals(fieldName, field.getName()),
"Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
if(field.getPossibleValueSourceName() != null)
{
assertCondition(errors, qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null,
"Unrecognized possibleValueSourceName " + field.getPossibleValueSourceName() + " in table " + tableName + " for field " + fieldName + ".");
}
});
}
});
}
validateBackends(qInstance, errors);
validateTables(qInstance, errors);
validateProcesses(qInstance, errors);
validateApps(qInstance, errors);
}
catch(Exception e)
{
@ -149,6 +107,232 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateBackends(QInstance qInstance, List<String> errors)
{
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getBackends()), "At least 1 backend must be defined."))
{
qInstance.getBackends().forEach((backendName, backend) ->
{
assertCondition(errors, Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
});
}
}
/*******************************************************************************
**
*******************************************************************************/
private void validateTables(QInstance qInstance, List<String> errors)
{
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getTables()),
"At least 1 table must be defined."))
{
qInstance.getTables().forEach((tableName, table) ->
{
assertCondition(errors, Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + ".");
validateAppChildHasValidParentAppName(qInstance, errors, table);
////////////////////////////////////////
// validate the backend for the table //
////////////////////////////////////////
if(assertCondition(errors, StringUtils.hasContent(table.getBackendName()),
"Missing backend name for table " + tableName + "."))
{
if(CollectionUtils.nullSafeHasContents(qInstance.getBackends()))
{
assertCondition(errors, qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + ".");
}
}
//////////////////////////////////
// validate fields in the table //
//////////////////////////////////
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(table.getFields()), "At least 1 field must be defined in table " + tableName + "."))
{
table.getFields().forEach((fieldName, field) ->
{
assertCondition(errors, Objects.equals(fieldName, field.getName()),
"Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
if(field.getPossibleValueSourceName() != null)
{
assertCondition(errors, qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null,
"Unrecognized possibleValueSourceName " + field.getPossibleValueSourceName() + " in table " + tableName + " for field " + fieldName + ".");
}
});
}
//////////////////////////////////////////
// validate field sections in the table //
//////////////////////////////////////////
Set<String> fieldNamesInSections = new HashSet<>();
QFieldSection tier1Section = null;
if(table.getSections() != null)
{
for(QFieldSection section : table.getSections())
{
validateSection(errors, table, section, fieldNamesInSections);
if(section.getTier().equals(Tier.T1))
{
assertCondition(errors, tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1");
tier1Section = section;
}
}
}
if(CollectionUtils.nullSafeHasContents(table.getFields()))
{
for(String fieldName : table.getFields().keySet())
{
assertCondition(errors, fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections.");
}
}
});
}
}
/*******************************************************************************
**
*******************************************************************************/
private void validateSection(List<String> errors, QTableMetaData table, QFieldSection section, Set<String> fieldNamesInSections)
{
assertCondition(errors, StringUtils.hasContent(section.getName()), "Missing a name for field section in table " + table.getName() + ".");
assertCondition(errors, StringUtils.hasContent(section.getLabel()), "Missing a label for field section in table " + table.getLabel() + ".");
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(section.getFieldNames()), "Table " + table.getName() + " section " + section.getName() + " does not have any fields."))
{
if(table.getFields() != null)
{
for(String fieldName : section.getFieldNames())
{
assertCondition(errors, table.getFields().containsKey(fieldName), "Table " + table.getName() + " section " + section.getName() + " specifies fieldName " + fieldName + ", which is not a field on this table.");
assertCondition(errors, !fieldNamesInSections.contains(fieldName), "Table " + table.getName() + " has field " + fieldName + " listed more than once in its field sections.");
fieldNamesInSections.add(fieldName);
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private void validateProcesses(QInstance qInstance, List<String> errors)
{
if(!CollectionUtils.nullSafeIsEmpty(qInstance.getProcesses()))
{
qInstance.getProcesses().forEach((processName, process) ->
{
assertCondition(errors, Objects.equals(processName, process.getName()), "Inconsistent naming for process: " + processName + "/" + process.getName() + ".");
validateAppChildHasValidParentAppName(qInstance, errors, process);
/////////////////////////////////////////////
// validate the table name for the process //
/////////////////////////////////////////////
if(process.getTableName() != null)
{
assertCondition(errors, qInstance.getTable(process.getTableName()) != null, "Unrecognized table " + process.getTableName() + " for process " + processName + ".");
}
///////////////////////////////////
// validate steps in the process //
///////////////////////////////////
if(assertCondition(errors, CollectionUtils.nullSafeHasContents(process.getStepList()), "At least 1 step must be defined in process " + processName + "."))
{
int index = 0;
for(QStepMetaData step : process.getStepList())
{
assertCondition(errors, StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName);
index++;
}
}
});
}
}
/*******************************************************************************
**
*******************************************************************************/
private void validateApps(QInstance qInstance, List<String> errors)
{
if(!CollectionUtils.nullSafeIsEmpty(qInstance.getApps()))
{
qInstance.getApps().forEach((appName, app) ->
{
assertCondition(errors, Objects.equals(appName, app.getName()), "Inconsistent naming for app: " + appName + "/" + app.getName() + ".");
validateAppChildHasValidParentAppName(qInstance, errors, app);
Set<String> appsVisited = new HashSet<>();
visitAppCheckingForCycles(app, appsVisited, errors);
if(app.getChildren() != null)
{
Set<String> childNames = new HashSet<>();
for(QAppChildMetaData child : app.getChildren())
{
assertCondition(errors, Objects.equals(appName, child.getParentAppName()), "Child " + child.getName() + " of app " + appName + " does not have its parent app properly set.");
assertCondition(errors, !childNames.contains(child.getName()), "App " + appName + " contains more than one child named " + child.getName());
childNames.add(child.getName());
}
}
});
}
}
/*******************************************************************************
** Check if an app's child list can recursively be traversed without finding a
** duplicate, which would indicate a cycle (e.g., an error)
*******************************************************************************/
private void visitAppCheckingForCycles(QAppMetaData app, Set<String> appsVisited, List<String> errors)
{
if(assertCondition(errors, !appsVisited.contains(app.getName()), "Circular app reference detected, involving " + app.getName()))
{
appsVisited.add(app.getName());
if(app.getChildren() != null)
{
for(QAppChildMetaData child : app.getChildren())
{
if(child instanceof QAppMetaData childApp)
{
visitAppCheckingForCycles(childApp, appsVisited, errors);
}
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private void validateAppChildHasValidParentAppName(QInstance qInstance, List<String> errors, QAppChildMetaData appChild)
{
if(appChild.getParentAppName() != null)
{
assertCondition(errors, qInstance.getApp(appChild.getParentAppName()) != null, "Unrecognized parent app " + appChild.getParentAppName() + " for " + appChild.getName() + ".");
}
}
/*******************************************************************************
** For the given input condition, if it's true, then we're all good (and return true).
** But if it's false, add the provided message to the list of errors (and return false,
** e.g., in case you need to stop evaluating rules to avoid exceptions).
*******************************************************************************/
private boolean assertCondition(List<String> errors, boolean condition, String message)
{
if(!condition)

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java
View File

@ -87,7 +87,7 @@ public abstract class AbstractActionInput
catch(QInstanceValidationException e)
{
LOG.warn(e);
throw (new IllegalArgumentException("QInstance failed validation" + e.getMessage()));
throw (new IllegalArgumentException("QInstance failed validation" + e.getMessage(), e));
}
}
}

51
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java
View File

@ -22,8 +22,11 @@
package com.kingsrook.qqq.backend.core.model.actions.metadata;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData;
@ -36,6 +39,9 @@ public class MetaDataOutput extends AbstractActionOutput
{
private Map<String, QFrontendTableMetaData> tables;
private Map<String, QFrontendProcessMetaData> processes;
private Map<String, QFrontendAppMetaData> apps;
private List<AppTreeNode> appTree;
@ -80,4 +86,49 @@ public class MetaDataOutput extends AbstractActionOutput
{
this.processes = processes;
}
/*******************************************************************************
** Getter for appTree
**
*******************************************************************************/
public List<AppTreeNode> getAppTree()
{
return appTree;
}
/*******************************************************************************
** Setter for appTree
**
*******************************************************************************/
public void setAppTree(List<AppTreeNode> appTree)
{
this.appTree = appTree;
}
/*******************************************************************************
** Getter for apps
**
*******************************************************************************/
public Map<String, QFrontendAppMetaData> getApps()
{
return apps;
}
/*******************************************************************************
** Setter for apps
**
*******************************************************************************/
public void setApps(Map<String, QFrontendAppMetaData> apps)
{
this.apps = apps;
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java
View File

@ -35,7 +35,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
** Output data container for the RunBackendStep action
**
*******************************************************************************/
public class RunBackendStepOutput extends AbstractActionOutput
public class RunBackendStepOutput extends AbstractActionOutput implements Serializable
{
private ProcessState processState;
private Exception exception; // todo - make optional

35
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
*******************************************************************************/
public class InsertInput extends AbstractTableActionInput
{
private QBackendTransaction transaction;
private List<QRecord> records;
@ -57,6 +59,39 @@ public class InsertInput extends AbstractTableActionInput
/*******************************************************************************
** Getter for transaction
**
*******************************************************************************/
public QBackendTransaction getTransaction()
{
return transaction;
}
/*******************************************************************************
** Setter for transaction
**
*******************************************************************************/
public void setTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
}
/*******************************************************************************
** Fluent setter for transaction
**
*******************************************************************************/
public InsertInput withTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
return (this);
}
/*******************************************************************************
** Getter for records
**

26
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java
View File

@ -32,12 +32,33 @@ import java.util.List;
*******************************************************************************/
public class QFilterCriteria implements Serializable
{
private String fieldName;
private QCriteriaOperator operator;
private String fieldName;
private QCriteriaOperator operator;
private List<Serializable> values;
/*******************************************************************************
**
*******************************************************************************/
public QFilterCriteria()
{
}
/*******************************************************************************
**
*******************************************************************************/
public QFilterCriteria(String fieldName, QCriteriaOperator operator, List<Serializable> values)
{
this.fieldName = fieldName;
this.operator = operator;
this.values = values;
}
/*******************************************************************************
** Getter for fieldName
**
@ -127,6 +148,7 @@ public class QFilterCriteria implements Serializable
}
/*******************************************************************************
** Setter for values
**

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java
View File

@ -23,10 +23,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -58,35 +56,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
@Override
public void addRecord(QRecord record)
{
if(!recordPipe.addRecord(record))
{
do
{
LOG.debug("Record pipe.add failed (due to full pipe). Blocking.");
SleepUtils.sleep(10, TimeUnit.MILLISECONDS);
}
while(!recordPipe.addRecord(record));
LOG.debug("Done blocking.");
}
}
/*******************************************************************************
**
*******************************************************************************/
private void blockIfPipeIsTooFull()
{
if(recordPipe.countAvailableRecords() >= 100_000)
{
LOG.info("Record pipe is kinda full. Blocking for a bit");
do
{
SleepUtils.sleep(10, TimeUnit.MILLISECONDS);
}
while(recordPipe.countAvailableRecords() >= 10_000);
LOG.info("Done blocking.");
}
recordPipe.addRecord(record);
}
@ -98,7 +68,6 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
public void addRecords(List<QRecord> records)
{
recordPipe.addRecords(records);
blockIfPipeIsTooFull();
}

68
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java Normal file
View File

@ -0,0 +1,68 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.data;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/*******************************************************************************
** Annotation to place onto fields in a QRecordEntity, to add additional attributes
** for propagating down into the corresponding QFieldMetaData
**
*******************************************************************************/
@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
public @interface QField
{
/*******************************************************************************
**
*******************************************************************************/
String label() default "";
/*******************************************************************************
**
*******************************************************************************/
String backendName() default "";
/*******************************************************************************
**
*******************************************************************************/
boolean isRequired() default false;
/*******************************************************************************
**
*******************************************************************************/
boolean isEditable() default true;
/*******************************************************************************
**
*******************************************************************************/
String displayFormat() default "";
//////////////////////////////////////////////////////////////////////////////////////////
// new attributes here likely need implementation in QFieldMetaData.constructFromGetter //
//////////////////////////////////////////////////////////////////////////////////////////
}

53
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java
View File

@ -30,6 +30,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -53,7 +54,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class QRecord implements Serializable
{
private String tableName;
private String tableName;
private String recordLabel;
private Map<String, Serializable> values = new LinkedHashMap<>();
private Map<String, String> displayValues = new LinkedHashMap<>();
private Map<String, Serializable> backendDetails = new LinkedHashMap<>();
@ -69,6 +72,7 @@ public class QRecord implements Serializable
}
/*******************************************************************************
**
*******************************************************************************/
@ -87,6 +91,7 @@ public class QRecord implements Serializable
public QRecord(QRecord record)
{
this.tableName = record.tableName;
this.recordLabel = record.recordLabel;
this.values = record.values;
this.displayValues = record.displayValues;
this.backendDetails = record.backendDetails;
@ -105,6 +110,16 @@ public class QRecord implements Serializable
/*******************************************************************************
**
*******************************************************************************/
public void setValue(QFieldMetaData field, Serializable value)
{
values.put(field.getName(), value);
}
/*******************************************************************************
**
*******************************************************************************/
@ -126,6 +141,7 @@ public class QRecord implements Serializable
/*******************************************************************************
**
*******************************************************************************/
@ -171,6 +187,39 @@ public class QRecord implements Serializable
/*******************************************************************************
** Getter for recordLabel
**
*******************************************************************************/
public String getRecordLabel()
{
return recordLabel;
}
/*******************************************************************************
** Setter for recordLabel
**
*******************************************************************************/
public void setRecordLabel(String recordLabel)
{
this.recordLabel = recordLabel;
}
/*******************************************************************************
** Fluent setter for recordLabel
**
*******************************************************************************/
public QRecord withRecordLabel(String recordLabel)
{
this.recordLabel = recordLabel;
return (this);
}
/*******************************************************************************
** Getter for values
**
@ -355,6 +404,7 @@ public class QRecord implements Serializable
}
/*******************************************************************************
** Getter for errors
**
@ -399,6 +449,7 @@ public class QRecord implements Serializable
}
/*******************************************************************************
** Convert this record to an QRecordEntity
*******************************************************************************/

43
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java
View File

@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.model.data;
import java.io.Serializable;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -104,7 +108,7 @@ public abstract class QRecordEntity
/*******************************************************************************
**
*******************************************************************************/
private static List<QRecordEntityField> getFieldList(Class<? extends QRecordEntity> c)
public static List<QRecordEntityField> getFieldList(Class<? extends QRecordEntity> c)
{
if(!fieldMapping.containsKey(c))
{
@ -114,10 +118,12 @@ public abstract class QRecordEntity
if(isGetter(possibleGetter))
{
Optional<Method> setter = getSetterForGetter(c, possibleGetter);
if(setter.isPresent())
{
String name = getFieldNameFromGetter(possibleGetter);
fieldList.add(new QRecordEntityField(name, possibleGetter, setter.get(), possibleGetter.getReturnType()));
String fieldName = getFieldNameFromGetter(possibleGetter);
Optional<QField> fieldAnnotation = getQFieldAnnotation(c, fieldName);
fieldList.add(new QRecordEntityField(fieldName, possibleGetter, setter.get(), possibleGetter.getReturnType(), fieldAnnotation.orElse(null)));
}
else
{
@ -132,6 +138,27 @@ public abstract class QRecordEntity
/*******************************************************************************
**
*******************************************************************************/
public static Optional<QField> getQFieldAnnotation(Class<? extends QRecordEntity> c, String fieldName)
{
try
{
Field field = c.getDeclaredField(fieldName);
return (Optional.ofNullable(field.getAnnotation(QField.class)));
}
catch(NoSuchFieldException e)
{
//////////////////////////////////////////
// ok, we just won't have an annotation //
//////////////////////////////////////////
}
return (Optional.empty());
}
/*******************************************************************************
**
*******************************************************************************/
@ -207,7 +234,15 @@ public abstract class QRecordEntity
|| returnType.equals(int.class)
|| returnType.equals(Boolean.class)
|| returnType.equals(boolean.class)
|| returnType.equals(BigDecimal.class));
|| returnType.equals(BigDecimal.class)
|| returnType.equals(Instant.class)
|| returnType.equals(LocalDate.class)
|| returnType.equals(LocalTime.class));
/////////////////////////////////////////////
// note - this list has implications upon: //
// - QFieldType.fromClass //
// - QRecordEntityField.convertValueType //
/////////////////////////////////////////////
}
}

82
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java
View File

@ -25,6 +25,9 @@ package com.kingsrook.qqq.backend.core.model.data;
import java.io.Serializable;
import java.lang.reflect.Method;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -38,18 +41,20 @@ public class QRecordEntityField
private final Method getter;
private final Method setter;
private final Class<?> type;
private final QField fieldAnnotation;
/*******************************************************************************
** Constructor.
*******************************************************************************/
public QRecordEntityField(String fieldName, Method getter, Method setter, Class<?> type)
public QRecordEntityField(String fieldName, Method getter, Method setter, Class<?> type, QField fieldAnnotation)
{
this.fieldName = fieldName;
this.getter = getter;
this.setter = setter;
this.type = type;
this.fieldAnnotation = fieldAnnotation;
}
@ -98,39 +103,72 @@ public class QRecordEntityField
/*******************************************************************************
** Getter for fieldAnnotation
**
*******************************************************************************/
public QField getFieldAnnotation()
{
return fieldAnnotation;
}
/*******************************************************************************
**
*******************************************************************************/
public Object convertValueType(Serializable value)
{
if(value == null)
try
{
return (null);
}
if(value == null)
{
return (null);
}
if(value.getClass().equals(type))
{
return (value);
}
if(value.getClass().equals(type))
{
return (value);
}
if(type.equals(String.class))
{
return (ValueUtils.getValueAsString(value));
}
if(type.equals(String.class))
{
return (ValueUtils.getValueAsString(value));
}
if(type.equals(Integer.class) || type.equals(int.class))
{
return (ValueUtils.getValueAsInteger(value));
}
if(type.equals(Integer.class) || type.equals(int.class))
{
return (ValueUtils.getValueAsInteger(value));
}
if(type.equals(Boolean.class) || type.equals(boolean.class))
{
return (ValueUtils.getValueAsBoolean(value));
}
if(type.equals(Boolean.class) || type.equals(boolean.class))
{
return (ValueUtils.getValueAsBoolean(value));
}
if(type.equals(BigDecimal.class))
if(type.equals(BigDecimal.class))
{
return (ValueUtils.getValueAsBigDecimal(value));
}
if(type.equals(LocalDate.class))
{
return (ValueUtils.getValueAsLocalDate(value));
}
if(type.equals(Instant.class))
{
return (ValueUtils.getValueAsInstant(value));
}
if(type.equals(LocalTime.class))
{
return (ValueUtils.getValueAsLocalTime(value));
}
}
catch(Exception e)
{
return (ValueUtils.getValueAsBigDecimal(value));
throw (new QValueException("Exception converting value [" + value + "] for field [" + fieldName + "]", e));
}
throw (new QValueException("Unhandled value type [" + type + "] for field [" + fieldName + "]"));

73
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java
View File

@ -24,10 +24,12 @@ package com.kingsrook.qqq.backend.core.model.metadata;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
@ -49,9 +51,13 @@ public class QInstance
private QAuthenticationMetaData authentication = null;
private Map<String, QTableMetaData> tables = new HashMap<>();
private Map<String, QPossibleValueSource<?>> possibleValueSources = new HashMap<>();
private Map<String, QProcessMetaData> processes = new HashMap<>();
////////////////////////////////////////////////////////////////////////////////////////////
// Important to use LinkedHashmap here, to preserve the order in which entries are added. //
////////////////////////////////////////////////////////////////////////////////////////////
private Map<String, QTableMetaData> tables = new LinkedHashMap<>();
private Map<String, QPossibleValueSource<?>> possibleValueSources = new LinkedHashMap<>();
private Map<String, QProcessMetaData> processes = new LinkedHashMap<>();
private Map<String, QAppMetaData> apps = new LinkedHashMap<>();
// todo - lock down the object (no more changes allowed) after it's been validated?
@ -171,6 +177,11 @@ public class QInstance
*******************************************************************************/
public QTableMetaData getTable(String name)
{
if(this.tables == null)
{
return (null);
}
return (this.tables.get(name));
}
@ -260,6 +271,40 @@ public class QInstance
/*******************************************************************************
**
*******************************************************************************/
public void addApp(QAppMetaData app)
{
this.addApp(app.getName(), app);
}
/*******************************************************************************
**
*******************************************************************************/
public void addApp(String name, QAppMetaData app)
{
if(this.apps.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second app with name: " + name));
}
this.apps.put(name, app);
}
/*******************************************************************************
**
*******************************************************************************/
public QAppMetaData getApp(String name)
{
return (this.apps.get(name));
}
/*******************************************************************************
** Getter for backends
**
@ -348,6 +393,28 @@ public class QInstance
/*******************************************************************************
** Getter for apps
**
*******************************************************************************/
public Map<String, QAppMetaData> getApps()
{
return apps;
}
/*******************************************************************************
** Setter for apps
**
*******************************************************************************/
public void setApps(Map<String, QAppMetaData> apps)
{
this.apps = apps;
}
/*******************************************************************************
** Getter for hasBeenValidated
**

40
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DisplayFormat.java Normal file
View File

@ -0,0 +1,40 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.fields;
/*******************************************************************************
**
*******************************************************************************/
public interface DisplayFormat
{
String DEFAULT = "%s";
String STRING = "%s";
String COMMAS = "%,d";
String DECIMAL1_COMMAS = "%,.1f";
String DECIMAL2_COMMAS = "%,.2f";
String DECIMAL3_COMMAS = "%,.3f";
String DECIMAL1 = "%.1f";
String DECIMAL2 = "%.2f";
String DECIMAL3 = "%.3f";
String CURRENCY = "$%,.2f";
}

87
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java
View File

@ -24,9 +24,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
import java.io.Serializable;
import java.lang.reflect.Method;
import java.util.Optional;
import com.github.hervian.reflection.Fun;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
@ -47,6 +50,7 @@ public class QFieldMetaData
// propose doing that in a secondary field, e.g., "onlyEditableOn=insert|update" //
///////////////////////////////////////////////////////////////////////////////////
private String displayFormat = "%s";
private Serializable defaultValue;
private String possibleValueSourceName;
@ -77,12 +81,59 @@ public class QFieldMetaData
** e.g., new QFieldMetaData(Order::getOrderNo).
*******************************************************************************/
public <T> QFieldMetaData(Fun.With1ParamAndVoid<T> getterRef) throws QException
{
Method getter = Fun.toMethod(getterRef);
constructFromGetter(getter);
}
/*******************************************************************************
** Initialize a fieldMetaData from a getter method from an entity
**
*******************************************************************************/
public <T> QFieldMetaData(Method getter) throws QException
{
constructFromGetter(getter);
}
/*******************************************************************************
** From a getter method, populate attributes in this field meta-data, including
** those from the @QField annotation on the field in the class, if present.
*******************************************************************************/
private void constructFromGetter(Method getter) throws QException
{
try
{
Method getter = Fun.toMethod(getterRef);
this.name = QRecordEntity.getFieldNameFromGetter(getter);
this.type = QFieldType.fromClass(getter.getReturnType());
@SuppressWarnings("unchecked")
Optional<QField> optionalFieldAnnotation = QRecordEntity.getQFieldAnnotation((Class<? extends QRecordEntity>) getter.getDeclaringClass(), this.name);
if(optionalFieldAnnotation.isPresent())
{
QField fieldAnnotation = optionalFieldAnnotation.get();
setIsRequired(fieldAnnotation.isRequired());
setIsEditable(fieldAnnotation.isEditable());
if(StringUtils.hasContent(fieldAnnotation.label()))
{
setLabel(fieldAnnotation.label());
}
if(StringUtils.hasContent(fieldAnnotation.backendName()))
{
setBackendName(fieldAnnotation.backendName());
}
if(StringUtils.hasContent(fieldAnnotation.displayFormat()))
{
setDisplayFormat(fieldAnnotation.displayFormat());
}
}
}
catch(QException qe)
{
@ -90,7 +141,7 @@ public class QFieldMetaData
}
catch(Exception e)
{
throw (new QException("Error constructing field from getterRef: " + getterRef, e));
throw (new QException("Error constructing field from getter method: " + getter.getName(), e));
}
}
@ -354,4 +405,36 @@ public class QFieldMetaData
return (this);
}
/*******************************************************************************
** Getter for displayFormat
**
*******************************************************************************/
public String getDisplayFormat()
{
return displayFormat;
}
/*******************************************************************************
** Setter for displayFormat
**
*******************************************************************************/
public void setDisplayFormat(String displayFormat)
{
this.displayFormat = displayFormat;
}
/*******************************************************************************
** Fluent setter for displayFormat
**
*******************************************************************************/
public QFieldMetaData withDisplayFormat(String displayFormat)
{
this.displayFormat = displayFormat;
return (this);
}
}

22
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java
View File

@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -35,8 +38,9 @@ public enum QFieldType
STRING,
INTEGER,
DECIMAL,
BOOLEAN,
DATE,
// TIME,
TIME,
DATE_TIME,
TEXT,
HTML,
@ -65,6 +69,22 @@ public enum QFieldType
{
return (DECIMAL);
}
if(c.equals(Instant.class))
{
return (DATE_TIME);
}
if(c.equals(LocalDate.class))
{
return (DATE);
}
if(c.equals(LocalTime.class))
{
return (TIME);
}
if(c.equals(Boolean.class))
{
return (BOOLEAN);
}
throw (new QException("Unrecognized class [" + c + "]"));
}

152
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNode.java Normal file
View File

@ -0,0 +1,152 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.frontend;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Frontend-version of objects that are parts of the app-hierarchy/tree.
** e.g., Tables, Processes, and Apps themselves (since they can be nested).
**
** These objects are organized into a tree - where each Node can have 0 or more
** other Nodes as children.
*******************************************************************************/
public class AppTreeNode
{
private AppTreeNodeType type;
private String name;
private String label;
private List<AppTreeNode> children;
private String iconName;
/*******************************************************************************
**
*******************************************************************************/
public AppTreeNode(QAppChildMetaData appChildMetaData)
{
this.name = appChildMetaData.getName();
this.label = appChildMetaData.getLabel();
if(appChildMetaData.getClass().equals(QTableMetaData.class))
{
this.type = AppTreeNodeType.TABLE;
}
else if(appChildMetaData.getClass().equals(QProcessMetaData.class))
{
this.type = AppTreeNodeType.PROCESS;
}
else if(appChildMetaData.getClass().equals(QAppMetaData.class))
{
this.type = AppTreeNodeType.APP;
children = new ArrayList<>();
}
else
{
throw (new IllegalStateException("Unrecognized class for app child meta data: " + appChildMetaData.getClass()));
}
if(appChildMetaData.getIcon() != null)
{
// todo - propagate icons from parents, if they aren't set here...
this.iconName = appChildMetaData.getIcon().getName();
}
}
/*******************************************************************************
** Getter for type
**
*******************************************************************************/
public AppTreeNodeType getType()
{
return type;
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Getter for label
**
*******************************************************************************/
public String getLabel()
{
return label;
}
/*******************************************************************************
** Getter for children
**
*******************************************************************************/
public List<AppTreeNode> getChildren()
{
return children;
}
/*******************************************************************************
** Getter for iconName
**
*******************************************************************************/
public String getIconName()
{
return iconName;
}
/*******************************************************************************
**
*******************************************************************************/
public void addChild(AppTreeNode childTreeNode)
{
if(children == null)
{
children = new ArrayList<>();
}
children.add(childTreeNode);
}
}

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNodeType.java Normal file
View File

@ -0,0 +1,33 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.frontend;
/*******************************************************************************
** Type for an Node in the an app tree.
*******************************************************************************/
public enum AppTreeNodeType
{
TABLE,
PROCESS,
APP
}

130
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java Normal file
View File

@ -0,0 +1,130 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.frontend;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
/*******************************************************************************
* Version of QAppMetaData that's meant for transmitting to a frontend.
*
*******************************************************************************/
@JsonInclude(Include.NON_NULL)
public class QFrontendAppMetaData
{
private String name;
private String label;
private List<AppTreeNode> children = new ArrayList<>();
private String iconName;
/*******************************************************************************
**
*******************************************************************************/
public QFrontendAppMetaData(QAppChildMetaData appChildMetaData)
{
this.name = appChildMetaData.getName();
this.label = appChildMetaData.getLabel();
if(appChildMetaData.getIcon() != null)
{
this.iconName = appChildMetaData.getIcon().getName();
}
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Getter for label
**
*******************************************************************************/
public String getLabel()
{
return label;
}
/*******************************************************************************
** Getter for children
**
*******************************************************************************/
public List<AppTreeNode> getChildren()
{
return children;
}
/*******************************************************************************
** Getter for iconName
**
*******************************************************************************/
public String getIconName()
{
return iconName;
}
/*******************************************************************************
** Setter for iconName
**
*******************************************************************************/
public void setIconName(String iconName)
{
this.iconName = iconName;
}
/*******************************************************************************
**
*******************************************************************************/
public void addChild(AppTreeNode childAppTreeNode)
{
if(children == null)
{
children = new ArrayList<>();
}
children.add(childAppTreeNode);
}
}

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java
View File

@ -45,6 +45,8 @@ public class QFrontendProcessMetaData
private String tableName;
private boolean isHidden;
private String iconName;
private List<QFrontendStepMetaData> frontendSteps;
//////////////////////////////////////////////////////////////////////////////////
@ -77,6 +79,11 @@ public class QFrontendProcessMetaData
frontendSteps = new ArrayList<>();
}
}
if(processMetaData.getIcon() != null)
{
this.iconName = processMetaData.getIcon().getName();
}
}
@ -148,12 +155,12 @@ public class QFrontendProcessMetaData
/*******************************************************************************
** Setter for isHidden
** Getter for iconName
**
*******************************************************************************/
public void setIsHidden(boolean isHidden)
public String getIconName()
{
this.isHidden = isHidden;
return iconName;
}
}

29
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java
View File

@ -23,10 +23,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -43,7 +45,10 @@ public class QFrontendTableMetaData
private boolean isHidden;
private String primaryKeyField;
private String iconName;
private Map<String, QFrontendFieldMetaData> fields;
private List<QFieldSection> sections;
//////////////////////////////////////////////////////////////////////////////////
// do not add setters. take values from the source-object in the constructor!! //
@ -68,6 +73,13 @@ public class QFrontendTableMetaData
{
this.fields.put(entry.getKey(), new QFrontendFieldMetaData(entry.getValue()));
}
this.sections = tableMetaData.getSections();
}
if(tableMetaData.getIcon() != null)
{
this.iconName = tableMetaData.getIcon().getName();
}
}
@ -117,6 +129,17 @@ public class QFrontendTableMetaData
/*******************************************************************************
** Getter for sections
**
*******************************************************************************/
public List<QFieldSection> getSections()
{
return sections;
}
/*******************************************************************************
** Getter for isHidden
**
@ -129,11 +152,11 @@ public class QFrontendTableMetaData
/*******************************************************************************
** Setter for isHidden
** Getter for iconName
**
*******************************************************************************/
public void setIsHidden(boolean isHidden)
public String getIconName()
{
this.isHidden = isHidden;
return iconName;
}
}

59
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppChildMetaData.java Normal file
View File

@ -0,0 +1,59 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.layout;
/*******************************************************************************
** Interface shared by meta-data objects which can be placed into an App.
** e.g., Tables, Processes, and Apps themselves (since they can be nested)
*******************************************************************************/
public interface QAppChildMetaData
{
/*******************************************************************************
**
*******************************************************************************/
void setParentAppName(String parentAppName);
/*******************************************************************************
**
*******************************************************************************/
String getParentAppName();
/*******************************************************************************
**
*******************************************************************************/
String getName();
/*******************************************************************************
**
*******************************************************************************/
String getLabel();
/*******************************************************************************
**
*******************************************************************************/
default QIcon getIcon()
{
return (null);
}
}

238
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java Normal file
View File

@ -0,0 +1,238 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.layout;
import java.util.ArrayList;
import java.util.List;
/*******************************************************************************
** MetaData definition of an App - an entity that organizes tables & processes
** and can be arranged hierarchically (e.g, apps can contain other apps).
*******************************************************************************/
public class QAppMetaData implements QAppChildMetaData
{
private String name;
private String label;
private List<QAppChildMetaData> children;
private String parentAppName;
private QIcon icon;
/*******************************************************************************
**
*******************************************************************************/
public QAppMetaData()
{
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public QAppMetaData withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for label
**
*******************************************************************************/
public String getLabel()
{
return label;
}
/*******************************************************************************
** Setter for label
**
*******************************************************************************/
public void setLabel(String label)
{
this.label = label;
}
/*******************************************************************************
** Fluent setter for label
**
*******************************************************************************/
public QAppMetaData withLabel(String label)
{
this.label = label;
return (this);
}
/*******************************************************************************
** Getter for children
**
*******************************************************************************/
public List<QAppChildMetaData> getChildren()
{
return children;
}
/*******************************************************************************
** Setter for children
**
*******************************************************************************/
public void setChildren(List<QAppChildMetaData> children)
{
this.children = children;
}
/*******************************************************************************
** Add a child to this app.
**
*******************************************************************************/
public void addChild(QAppChildMetaData child)
{
if(this.children == null)
{
this.children = new ArrayList<>();
}
this.children.add(child);
child.setParentAppName(this.getName());
}
/*******************************************************************************
** Fluently add a child to this app.
**
*******************************************************************************/
public QAppMetaData withChild(QAppChildMetaData child)
{
addChild(child);
return (this);
}
/*******************************************************************************
** Fluent setter for children
**
*******************************************************************************/
public QAppMetaData withChildren(List<QAppChildMetaData> children)
{
this.children = children;
return (this);
}
/*******************************************************************************
** Getter for parentAppName
**
*******************************************************************************/
@Override
public String getParentAppName()
{
return parentAppName;
}
/*******************************************************************************
** Setter for parentAppName
**
*******************************************************************************/
@Override
public void setParentAppName(String parentAppName)
{
this.parentAppName = parentAppName;
}
/*******************************************************************************
** Getter for icon
**
*******************************************************************************/
public QIcon getIcon()
{
return icon;
}
/*******************************************************************************
** Setter for icon
**
*******************************************************************************/
public void setIcon(QIcon icon)
{
this.icon = icon;
}
/*******************************************************************************
** Fluent setter for icon
**
*******************************************************************************/
public QAppMetaData withIcon(QIcon icon)
{
this.icon = icon;
return (this);
}
}

91
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java Normal file
View File

@ -0,0 +1,91 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.layout;
/*******************************************************************************
** Icon to show associated with an App, Table, Process, etc.
**
** Currently, name here must be a reference from https://fonts.google.com/icons
** e.g., local_shipping for https://fonts.google.com/icons?selected=Material+Symbols+Outlined:local_shipping
**
** Future may allow something like a "namespace", and/or multiple icons for
** use in different frontends, etc.
*******************************************************************************/
public class QIcon
{
private String name;
/*******************************************************************************
**
*******************************************************************************/
public QIcon()
{
}
/*******************************************************************************
**
*******************************************************************************/
public QIcon(String name)
{
this.name = name;
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public QIcon withName(String name)
{
this.name = name;
return (this);
}
}

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java
View File

@ -26,13 +26,15 @@ import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
/*******************************************************************************
** Meta-Data to define a process in a QQQ instance.
**
*******************************************************************************/
public class QProcessMetaData
public class QProcessMetaData implements QAppChildMetaData
{
private String name;
private String label;
@ -41,6 +43,9 @@ public class QProcessMetaData
private List<QStepMetaData> stepList;
private String parentAppName;
private QIcon icon;
/*******************************************************************************
@ -293,4 +298,62 @@ public class QProcessMetaData
return (this);
}
/*******************************************************************************
** Getter for parentAppName
**
*******************************************************************************/
@Override
public String getParentAppName()
{
return parentAppName;
}
/*******************************************************************************
** Setter for parentAppName
**
*******************************************************************************/
@Override
public void setParentAppName(String parentAppName)
{
this.parentAppName = parentAppName;
}
/*******************************************************************************
** Getter for icon
**
*******************************************************************************/
public QIcon getIcon()
{
return icon;
}
/*******************************************************************************
** Setter for icon
**
*******************************************************************************/
public void setIcon(QIcon icon)
{
this.icon = icon;
}
/*******************************************************************************
** Fluent setter for icon
**
*******************************************************************************/
public QProcessMetaData withIcon(QIcon icon)
{
this.icon = icon;
return (this);
}
}

234
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java Normal file
View File

@ -0,0 +1,234 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
/*******************************************************************************
** A section of fields - a logical grouping.
*******************************************************************************/
public class QFieldSection
{
private String name;
private String label;
private Tier tier;
private List<String> fieldNames;
private QIcon icon;
/*******************************************************************************
**
*******************************************************************************/
public QFieldSection()
{
}
/*******************************************************************************
**
*******************************************************************************/
public QFieldSection(String name, String label, QIcon icon, Tier tier, List<String> fieldNames)
{
this.name = name;
this.label = label;
this.icon = icon;
this.tier = tier;
this.fieldNames = fieldNames;
}
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public QFieldSection withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for label
**
*******************************************************************************/
public String getLabel()
{
return label;
}
/*******************************************************************************
** Setter for label
**
*******************************************************************************/
public void setLabel(String label)
{
this.label = label;
}
/*******************************************************************************
** Fluent setter for label
**
*******************************************************************************/
public QFieldSection withLabel(String label)
{
this.label = label;
return (this);
}
/*******************************************************************************
** Getter for tier
**
*******************************************************************************/
public Tier getTier()
{
return tier;
}
/*******************************************************************************
** Setter for tier
**
*******************************************************************************/
public void setTier(Tier tier)
{
this.tier = tier;
}
/*******************************************************************************
** Fluent setter for tier
**
*******************************************************************************/
public QFieldSection withTier(Tier tier)
{
this.tier = tier;
return (this);
}
/*******************************************************************************
** Getter for fieldNames
**
*******************************************************************************/
public List<String> getFieldNames()
{
return fieldNames;
}
/*******************************************************************************
** Setter for fieldNames
**
*******************************************************************************/
public void setFieldNames(List<String> fieldNames)
{
this.fieldNames = fieldNames;
}
/*******************************************************************************
** Fluent setter for fieldNames
**
*******************************************************************************/
public QFieldSection withFieldNames(List<String> fieldNames)
{
this.fieldNames = fieldNames;
return (this);
}
/*******************************************************************************
** Getter for icon
**
*******************************************************************************/
public QIcon getIcon()
{
return icon;
}
/*******************************************************************************
** Setter for icon
**
*******************************************************************************/
public void setIcon(QIcon icon)
{
this.icon = icon;
}
/*******************************************************************************
** Fluent setter for icon
**
*******************************************************************************/
public QFieldSection withIcon(QIcon icon)
{
this.icon = icon;
return (this);
}
}

217
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java
View File

@ -23,20 +23,26 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntityField;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
/*******************************************************************************
** Meta-Data to define a table in a QQQ instance.
**
*******************************************************************************/
public class QTableMetaData implements Serializable
public class QTableMetaData implements QAppChildMetaData, Serializable
{
private String name;
private String label;
@ -59,6 +65,14 @@ public class QTableMetaData implements Serializable
private Map<String, QCodeReference> customizers;
private String parentAppName;
private QIcon icon;
private String recordLabelFormat;
private List<String> recordLabelFields;
private List<QFieldSection> sections;
/*******************************************************************************
@ -91,6 +105,22 @@ public class QTableMetaData implements Serializable
/*******************************************************************************
**
*******************************************************************************/
public QTableMetaData withFieldsFromEntity(Class<? extends QRecordEntity> entityClass) throws QException
{
List<QRecordEntityField> recordEntityFieldList = QRecordEntity.getFieldList(entityClass);
for(QRecordEntityField recordEntityField : recordEntityFieldList)
{
QFieldMetaData field = new QFieldMetaData(recordEntityField.getGetter());
addField(field);
}
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@ -434,4 +464,189 @@ public class QTableMetaData implements Serializable
return (this);
}
/*******************************************************************************
** Getter for parentAppName
**
*******************************************************************************/
@Override
public String getParentAppName()
{
return parentAppName;
}
/*******************************************************************************
** Setter for parentAppName
**
*******************************************************************************/
@Override
public void setParentAppName(String parentAppName)
{
this.parentAppName = parentAppName;
}
/*******************************************************************************
** Getter for icon
**
*******************************************************************************/
public QIcon getIcon()
{
return icon;
}
/*******************************************************************************
** Setter for icon
**
*******************************************************************************/
public void setIcon(QIcon icon)
{
this.icon = icon;
}
/*******************************************************************************
** Fluent setter for icon
**
*******************************************************************************/
public QTableMetaData withIcon(QIcon icon)
{
this.icon = icon;
return (this);
}
/*******************************************************************************
** Getter for recordLabelFormat
**
*******************************************************************************/
public String getRecordLabelFormat()
{
return recordLabelFormat;
}
/*******************************************************************************
** Setter for recordLabelFormat
**
*******************************************************************************/
public void setRecordLabelFormat(String recordLabelFormat)
{
this.recordLabelFormat = recordLabelFormat;
}
/*******************************************************************************
** Fluent setter for recordLabelFormat
**
*******************************************************************************/
public QTableMetaData withRecordLabelFormat(String recordLabelFormat)
{
this.recordLabelFormat = recordLabelFormat;
return (this);
}
/*******************************************************************************
** Getter for recordLabelFields
**
*******************************************************************************/
public List<String> getRecordLabelFields()
{
return recordLabelFields;
}
/*******************************************************************************
** Setter for recordLabelFields
**
*******************************************************************************/
public void setRecordLabelFields(List<String> recordLabelFields)
{
this.recordLabelFields = recordLabelFields;
}
/*******************************************************************************
** Fluent setter for recordLabelFields
**
*******************************************************************************/
public QTableMetaData withRecordLabelFields(List<String> recordLabelFields)
{
this.recordLabelFields = recordLabelFields;
return (this);
}
/*******************************************************************************
** Getter for sections
**
*******************************************************************************/
public List<QFieldSection> getSections()
{
return sections;
}
/*******************************************************************************
** Setter for sections
**
*******************************************************************************/
public void setSections(List<QFieldSection> sections)
{
this.sections = sections;
}
/*******************************************************************************
** Fluent setter for sections
**
*******************************************************************************/
public QTableMetaData withSections(List<QFieldSection> fieldSections)
{
this.sections = fieldSections;
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public void addSection(QFieldSection fieldSection)
{
if(this.sections == null)
{
this.sections = new ArrayList<>();
}
this.sections.add(fieldSection);
}
/*******************************************************************************
**
*******************************************************************************/
public QTableMetaData withSection(QFieldSection fieldSection)
{
addSection(fieldSection);
return (this);
}
}

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Tier.java Normal file
View File

@ -0,0 +1,33 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.tables;
/*******************************************************************************
**
*******************************************************************************/
public enum Tier
{
T1,
T2,
T3
}

91
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java
View File

@ -57,20 +57,25 @@ import org.json.JSONObject;
*******************************************************************************/
public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
{
private static final Logger logger = LogManager.getLogger(Auth0AuthenticationModule.class);
private static final Logger LOG = LogManager.getLogger(Auth0AuthenticationModule.class);
private static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 300;
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
// 30 minutes - ideally this would be lower, but right now we've been dealing with re-validation issues... //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
public static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 1800;
public static final String AUTH0_ID_TOKEN_KEY = "sessionId";
public static final String TOKEN_NOT_PROVIDED_ERROR = "Id Token was not provided";
public static final String COULD_NOT_DECODE_ERROR = "Unable to decode id token";
public static final String EXPIRED_TOKEN_ERROR = "Token has expired";
public static final String INVALID_TOKEN_ERROR = "An invalid token was provided";
public static final String COULD_NOT_DECODE_ERROR = "Unable to decode id token";
public static final String EXPIRED_TOKEN_ERROR = "Token has expired";
public static final String INVALID_TOKEN_ERROR = "An invalid token was provided";
private Instant now;
/*******************************************************************************
**
*******************************************************************************/
@ -83,7 +88,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
String idToken = context.get(AUTH0_ID_TOKEN_KEY);
if(idToken == null)
{
logger.warn(TOKEN_NOT_PROVIDED_ERROR);
LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
}
@ -97,7 +102,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// then call method to check more session validity //
/////////////////////////////////////////////////////
QSession qSession = buildQSessionFromToken(idToken);
if(isSessionValid(qSession))
if(isSessionValid(qInstance, qSession))
{
return (qSession);
}
@ -112,7 +117,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// put now into state so we dont check until next interval passes //
///////////////////////////////////////////////////////////////////
StateProviderInterface spi = getStateProvider();
Auth0StateKey key = new Auth0StateKey(qSession.getIdReference());
Auth0StateKey key = new Auth0StateKey(qSession.getIdReference());
spi.put(key, Instant.now());
return (qSession);
@ -122,12 +127,12 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
////////////////////////////////
// could not decode the token //
////////////////////////////////
logger.warn(COULD_NOT_DECODE_ERROR, jde);
LOG.warn(COULD_NOT_DECODE_ERROR, jde);
throw (new QAuthenticationException(COULD_NOT_DECODE_ERROR));
}
catch(TokenExpiredException tee)
{
logger.info(EXPIRED_TOKEN_ERROR, tee);
LOG.info(EXPIRED_TOKEN_ERROR, tee);
throw (new QAuthenticationException(EXPIRED_TOKEN_ERROR));
}
catch(JWTVerificationException | JwkException jve)
@ -135,7 +140,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
///////////////////////////////////////////
// token had invalid signature or claims //
///////////////////////////////////////////
logger.warn(INVALID_TOKEN_ERROR, jve);
LOG.warn(INVALID_TOKEN_ERROR, jve);
throw (new QAuthenticationException(INVALID_TOKEN_ERROR));
}
catch(Exception e)
@ -144,7 +149,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// ¯\_(ツ)_/¯ //
////////////////
String message = "An unknown error occurred";
logger.error(message, e);
LOG.error(message, e);
throw (new QAuthenticationException(message));
}
}
@ -155,16 +160,16 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
**
*******************************************************************************/
@Override
public boolean isSessionValid(QSession session)
public boolean isSessionValid(QInstance instance, QSession session)
{
if(session == null)
{
return (false);
}
StateProviderInterface spi = getStateProvider();
Auth0StateKey key = new Auth0StateKey(session.getIdReference());
Optional<Instant> lastTimeCheckedOptional = spi.get(Instant.class, key);
StateProviderInterface spi = getStateProvider();
Auth0StateKey key = new Auth0StateKey(session.getIdReference());
Optional<Instant> lastTimeCheckedOptional = spi.get(Instant.class, key);
if(lastTimeCheckedOptional.isPresent())
{
Instant lastTimeChecked = lastTimeCheckedOptional.get();
@ -174,7 +179,28 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// - so this is basically saying, if the time between the last time we checked the token and //
// right now is more than ID_TOKEN_VALIDATION_INTERVAL_SECTIONS, then session needs revalidated //
///////////////////////////////////////////////////////////////////////////////////////////////////
return (Duration.between(lastTimeChecked, Instant.now()).compareTo(Duration.ofSeconds(ID_TOKEN_VALIDATION_INTERVAL_SECONDS)) < 0);
if(Duration.between(lastTimeChecked, Instant.now()).compareTo(Duration.ofSeconds(ID_TOKEN_VALIDATION_INTERVAL_SECONDS)) < 0)
{
return (true);
}
try
{
LOG.debug("Re-validating token due to validation interval being passed: " + session.getIdReference());
revalidateToken(instance, session.getIdReference());
//////////////////////////////////////////////////////////////////
// update the timestamp in state provider, to avoid re-checking //
//////////////////////////////////////////////////////////////////
spi.put(key, Instant.now());
return (true);
}
catch(Exception e)
{
LOG.warn(INVALID_TOKEN_ERROR, e);
return (false);
}
}
return (false);
@ -190,10 +216,10 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
{
Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
DecodedJWT jwt = JWT.decode(idToken);
JwkProvider provider = new UrlJwkProvider(metaData.getBaseUrl());
Jwk jwk = provider.get(jwt.getKeyId());
Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null);
DecodedJWT jwt = JWT.decode(idToken);
JwkProvider provider = new UrlJwkProvider(metaData.getBaseUrl());
Jwk jwk = provider.get(jwt.getKeyId());
Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null);
JWTVerifier verifier = JWT.require(algorithm)
.withIssuer(metaData.getBaseUrl())
.build();
@ -217,20 +243,31 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
////////////////////////////////////
// decode and extract the payload //
////////////////////////////////////
DecodedJWT jwt = JWT.decode(idToken);
Base64.Decoder decoder = Base64.getUrlDecoder();
String payloadString = new String(decoder.decode(jwt.getPayload()));
JSONObject payload = new JSONObject(payloadString);
DecodedJWT jwt = JWT.decode(idToken);
Base64.Decoder decoder = Base64.getUrlDecoder();
String payloadString = new String(decoder.decode(jwt.getPayload()));
JSONObject payload = new JSONObject(payloadString);
QUser qUser = new QUser();
qUser.setFullName(payload.getString("name"));
if(payload.has("name"))
{
qUser.setFullName(payload.getString("name"));
}
else
{
qUser.setFullName("Unknown");
}
if(payload.has("email"))
{
qUser.setIdReference(payload.getString("email"));
}
else
{
qUser.setIdReference(payload.getString("nickname"));
if(payload.has("sub"))
{
qUser.setIdReference(payload.getString("sub"));
}
}
QSession qSession = new QSession();

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java
View File

@ -66,7 +66,7 @@ public class FullyAnonymousAuthenticationModule implements QAuthenticationModule
**
*******************************************************************************/
@Override
public boolean isSessionValid(QSession session)
public boolean isSessionValid(QInstance instance, QSession session)
{
return session != null;
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java
View File

@ -63,7 +63,7 @@ public class MockAuthenticationModule implements QAuthenticationModuleInterface
**
*******************************************************************************/
@Override
public boolean isSessionValid(QSession session)
public boolean isSessionValid(QInstance instance, QSession session)
{
if(session == null)
{

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java
View File

@ -43,5 +43,5 @@ public interface QAuthenticationModuleInterface
/*******************************************************************************
**
*******************************************************************************/
boolean isSessionValid(QSession session);
boolean isSessionValid(QInstance instance, QSession session);
}

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@ -40,6 +41,8 @@ public class BasicETLExtractFunction implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLExtractFunction.class);
private RecordPipe recordPipe = null;
/*******************************************************************************
@ -64,10 +67,35 @@ public class BasicETLExtractFunction implements BackendStep
// queryRequest.setFilter(JsonUtils.toObject(filter, QQueryFilter.class));
// }
//////////////////////////////////////////////////////////////////////
// if the caller gave us a record pipe, pass it to the query action //
//////////////////////////////////////////////////////////////////////
if (recordPipe != null)
{
queryInput.setRecordPipe(recordPipe);
}
QueryAction queryAction = new QueryAction();
QueryOutput queryOutput = queryAction.execute(queryInput);
runBackendStepOutput.setRecords(queryOutput.getRecords());
LOG.info("Query on table " + tableName + " produced " + queryOutput.getRecords().size() + " records.");
if (recordPipe == null)
{
////////////////////////////////////////////////////////////////////////////
// only return the records (and log about them) if there's no record pipe //
////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.setRecords(queryOutput.getRecords());
LOG.info("Query on table " + tableName + " produced " + queryOutput.getRecords().size() + " records.");
}
}
/*******************************************************************************
** Setter for recordPipe
**
*******************************************************************************/
public void setRecordPipe(RecordPipe recordPipe)
{
this.recordPipe = recordPipe;
}
}

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -44,6 +45,9 @@ public class BasicETLLoadFunction implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLLoadFunction.class);
private QBackendTransaction transaction;
private boolean returnStoredRecords = true;
/*******************************************************************************
@ -86,10 +90,15 @@ public class BasicETLLoadFunction implements BackendStep
insertInput.setSession(runBackendStepInput.getSession());
insertInput.setTableName(table);
insertInput.setRecords(page);
insertInput.setTransaction(transaction);
InsertAction insertAction = new InsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
outputRecords.addAll(insertOutput.getRecords());
if(returnStoredRecords)
{
outputRecords.addAll(insertOutput.getRecords());
}
recordsInserted += insertOutput.getRecords().size();
}
@ -97,4 +106,25 @@ public class BasicETLLoadFunction implements BackendStep
runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsInserted);
}
/*******************************************************************************
** Setter for transaction
**
*******************************************************************************/
public void setTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
}
/*******************************************************************************
** Setter for returnStoredRecords
**
*******************************************************************************/
public void setReturnStoredRecords(boolean returnStoredRecords)
{
this.returnStoredRecords = returnStoredRecords;
}
}

249
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java Normal file
View File

@ -0,0 +1,249 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLExtractFunction;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLLoadFunction;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLTransformFunction;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Backend step to do a streamed ETL
*******************************************************************************/
public class StreamedETLBackendStep implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(StreamedETLBackendStep.class);
private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000;
private static final int MAX_SLEEP_MS = 1000;
private static final int INIT_SLEEP_MS = 10;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
QBackendTransaction transaction = openTransaction(runBackendStepInput);
try
{
RecordPipe recordPipe = new RecordPipe();
BasicETLExtractFunction basicETLExtractFunction = new BasicETLExtractFunction();
basicETLExtractFunction.setRecordPipe(recordPipe);
//////////////////////////////////////////
// run the query action as an async job //
//////////////////////////////////////////
AsyncJobManager asyncJobManager = new AsyncJobManager();
String queryJobUUID = asyncJobManager.startJob("StreamedETL>QueryAction", (status) ->
{
basicETLExtractFunction.run(runBackendStepInput, runBackendStepOutput);
return (runBackendStepOutput);
});
LOG.info("Started query job [" + queryJobUUID + "] for streamed ETL");
AsyncJobState queryJobState = AsyncJobState.RUNNING;
AsyncJobStatus asyncJobStatus = null;
long recordCount = 0;
int nextSleepMillis = INIT_SLEEP_MS;
long lastReceivedRecordsAt = System.currentTimeMillis();
long jobStartTime = System.currentTimeMillis();
while(queryJobState.equals(AsyncJobState.RUNNING))
{
if(recordPipe.countAvailableRecords() == 0)
{
///////////////////////////////////////////////////////////
// if the pipe is empty, sleep to let the producer work. //
// todo - smarter sleep? like get notified vs. sleep? //
///////////////////////////////////////////////////////////
LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work");
SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS);
long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt;
if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS)
{
throw (new QException("Query action appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago)."));
}
}
else
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. //
////////////////////////////////////////////////////////////////////////////////////////////////////////
lastReceivedRecordsAt = System.currentTimeMillis();
nextSleepMillis = INIT_SLEEP_MS;
recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction);
LOG.info(String.format("Processed %,d records so far", recordCount));
}
////////////////////////////////////
// refresh the query job's status //
////////////////////////////////////
Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(queryJobUUID);
if(optionalAsyncJobStatus.isEmpty())
{
/////////////////////////////////////////////////
// todo - ... maybe some version of try-again? //
/////////////////////////////////////////////////
throw (new QException("Could not get status of report query job [" + queryJobUUID + "]"));
}
asyncJobStatus = optionalAsyncJobStatus.get();
queryJobState = asyncJobStatus.getState();
}
LOG.info("Query job [" + queryJobUUID + "] for ETL completed with status: " + asyncJobStatus);
/////////////////////////////////////////
// propagate errors from the query job //
/////////////////////////////////////////
if(asyncJobStatus.getState().equals(AsyncJobState.ERROR))
{
throw (new QException("Query job failed with an error", asyncJobStatus.getCaughtException()));
}
//////////////////////////////////////////////////////
// send the final records to transform & load steps //
//////////////////////////////////////////////////////
recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction);
/////////////////////
// commit the work //
/////////////////////
transaction.commit();
long reportEndTime = System.currentTimeMillis();
LOG.info(String.format("Processed %,d records", recordCount)
+ String.format(" at end of ETL job in %,d ms (%.2f records/second).", (reportEndTime - jobStartTime), 1000d * (recordCount / (.001d + (reportEndTime - jobStartTime)))));
runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount);
}
catch(Exception e)
{
////////////////////////////////////////////////////////////////////////////////
// rollback the work, then re-throw the error for up-stream to catch & report //
////////////////////////////////////////////////////////////////////////////////
transaction.rollback();
throw (e);
}
finally
{
////////////////////////////////////////////////////////////
// always close our transactions (e.g., jdbc connections) //
////////////////////////////////////////////////////////////
transaction.close();
}
}
/*******************************************************************************
**
*******************************************************************************/
private QBackendTransaction openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance());
insertInput.setSession(runBackendStepInput.getSession());
insertInput.setTableName(runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE));
return new InsertAction().openTransaction(insertInput);
}
/*******************************************************************************
**
*******************************************************************************/
private int consumeRecordsFromPipe(RecordPipe recordPipe, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, QBackendTransaction transaction) throws QException
{
List<QRecord> qRecords = recordPipe.consumeAvailableRecords();
preTransform(qRecords, runBackendStepInput, runBackendStepOutput);
runBackendStepInput.setRecords(qRecords);
new BasicETLTransformFunction().run(runBackendStepInput, runBackendStepOutput);
postTransform(qRecords, runBackendStepInput, runBackendStepOutput);
runBackendStepInput.setRecords(runBackendStepOutput.getRecords());
BasicETLLoadFunction basicETLLoadFunction = new BasicETLLoadFunction();
basicETLLoadFunction.setReturnStoredRecords(false);
basicETLLoadFunction.setTransaction(transaction);
basicETLLoadFunction.run(runBackendStepInput, runBackendStepOutput);
return (qRecords.size());
}
/*******************************************************************************
** Customization point for subclasses of this step.
*******************************************************************************/
protected void preTransform(List<QRecord> qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
////////////////////////
// noop in base class //
////////////////////////
}
/*******************************************************************************
** Customization point for subclasses of this step.
*******************************************************************************/
protected void postTransform(List<QRecord> qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
////////////////////////
// noop in base class //
////////////////////////
}
}

75
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java Normal file
View File

@ -0,0 +1,75 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
/*******************************************************************************
** Definition for Streamed ETL process.
*******************************************************************************/
public class StreamedETLProcess
{
public static final String PROCESS_NAME = "etl.streamed";
public static final String FUNCTION_NAME_ETL = "streamedETL";
public static final String FIELD_SOURCE_TABLE = "sourceTable";
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
public static final String FIELD_MAPPING_JSON = "mappingJSON";
public static final String FIELD_RECORD_COUNT = "recordCount";
/*******************************************************************************
**
*******************************************************************************/
public QProcessMetaData defineProcessMetaData()
{
QStepMetaData etlFunction = new QBackendStepMetaData()
.withName(FUNCTION_NAME_ETL)
.withCode(new QCodeReference()
.withName(StreamedETLBackendStep.class.getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
.withField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING))
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)))
.withOutputMetaData(new QFunctionOutputMetaData()
.addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));
return new QProcessMetaData()
.withName(PROCESS_NAME)
.addStep(etlFunction);
}
}

16
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ListingHash.java
View File

@ -41,7 +41,7 @@ public class ListingHash<K, V> implements Map<K, List<V>>, Serializable
{
public static final long serialVersionUID = 0L;
private HashMap<K, List<V>> hashMap = null;
private Map<K, List<V>> hashMap = null;
@ -51,7 +51,19 @@ public class ListingHash<K, V> implements Map<K, List<V>>, Serializable
*******************************************************************************/
public ListingHash()
{
this.hashMap = new HashMap<K, List<V>>();
this.hashMap = new HashMap<>();
}
/*******************************************************************************
** Constructor where you can supply a source map (e.g., if you want a specific
** Map type (like LinkedHashMap), or with pre-values
**
*******************************************************************************/
public ListingHash(Map<K, List<V>> sourceMap)
{
this.hashMap = sourceMap;
}

161
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java
View File

@ -22,12 +22,17 @@
package com.kingsrook.qqq.backend.core.utils;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Calendar;
import java.util.List;
import java.util.TimeZone;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
@ -37,7 +42,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QValueException;
*******************************************************************************/
public class ValueUtils
{
private static final DateTimeFormatter localDateDefaultFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final DateTimeFormatter dateTimeFormatter_yyyyMMddWithDashes = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final DateTimeFormatter dateTimeFormatter_MdyyyyWithSlashes = DateTimeFormatter.ofPattern("M/d/yyyy");
@ -174,9 +180,13 @@ public class ValueUtils
}
else
{
throw (new IllegalArgumentException("Unsupported class " + value.getClass().getName() + " for converting to Integer."));
throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to Integer."));
}
}
catch(QValueException qve)
{
throw (qve);
}
catch(Exception e)
{
throw (new QValueException("Value [" + value + "] could not be converted to an Integer.", e));
@ -212,8 +222,8 @@ public class ValueUtils
}
else if(value instanceof Calendar c)
{
TimeZone tz = c.getTimeZone();
ZoneId zid = (tz == null) ? ZoneId.systemDefault() : tz.toZoneId();
TimeZone tz = c.getTimeZone();
ZoneId zid = (tz == null) ? ZoneId.systemDefault() : tz.toZoneId();
return LocalDateTime.ofInstant(c.toInstant(), zid).toLocalDate();
}
else if(value instanceof LocalDateTime ldt)
@ -227,21 +237,47 @@ public class ValueUtils
return (null);
}
return LocalDate.parse(s, localDateDefaultFormatter);
return tryLocalDateParsers(s);
}
else
{
throw (new IllegalArgumentException("Unsupported class " + value.getClass().getName() + " for converting to LocalDate."));
throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to LocalDate."));
}
}
catch(QValueException qve)
{
throw (qve);
}
catch(Exception e)
{
throw (new QValueException("Value [" + value + "] could not be converted to an LocalDate.", e));
throw (new QValueException("Value [" + value + "] could not be converted to a LocalDate.", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
private static LocalDate tryLocalDateParsers(String s)
{
DateTimeParseException lastException = null;
for(DateTimeFormatter dateTimeFormatter : List.of(dateTimeFormatter_yyyyMMddWithDashes, dateTimeFormatter_MdyyyyWithSlashes))
{
try
{
return LocalDate.parse(s, dateTimeFormatter);
}
catch(DateTimeParseException dtpe)
{
lastException = dtpe;
}
}
throw (new QValueException("Could not parse value [" + s + "] to a local date", lastException));
}
/*******************************************************************************
** Type-safely make a BigDecimal from any Object.
** null and empty-string inputs return null.
@ -305,13 +341,120 @@ public class ValueUtils
}
else
{
throw (new IllegalArgumentException("Unsupported class " + value.getClass().getName() + " for converting to BigDecimal."));
throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to BigDecimal."));
}
}
catch(QValueException qve)
{
throw (qve);
}
catch(Exception e)
{
throw (new QValueException("Value [" + value + "] could not be converted to an BigDecimal.", e));
throw (new QValueException("Value [" + value + "] could not be converted to a BigDecimal.", e));
}
}
/*******************************************************************************
** Type-safely make an Instant from any Object.
** null and empty-string inputs return null.
** We may throw if the input can't be converted to a Instant
*******************************************************************************/
public static Instant getValueAsInstant(Object value)
{
try
{
if(value == null)
{
return (null);
}
else if(value instanceof Instant i)
{
return (i);
}
else if(value instanceof java.sql.Date d)
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - in the jdk, this method throws UnsupportedOperationException (because of the lack of time in sql Dates) //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
return d.toInstant();
}
else if(value instanceof java.util.Date d)
{
return d.toInstant();
}
else if(value instanceof Calendar c)
{
return (c.toInstant());
}
else if(value instanceof LocalDateTime ldt)
{
ZoneId zoneId = ZoneId.systemDefault();
return ldt.toInstant(zoneId.getRules().getOffset(ldt));
}
else if(value instanceof String s)
{
if(!StringUtils.hasContent(s))
{
return (null);
}
return Instant.parse(s);
}
else
{
throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to Instant."));
}
}
catch(QValueException qve)
{
throw (qve);
}
catch(Exception e)
{
throw (new QValueException("Value [" + value + "] could not be converted to a Instant.", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
public static Object getValueAsLocalTime(Serializable value)
{
try
{
if(value == null)
{
return (null);
}
else if(value instanceof LocalTime lt)
{
return (lt);
}
else if(value instanceof String s)
{
if(!StringUtils.hasContent(s))
{
return (null);
}
return LocalTime.parse(s);
}
else
{
throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to LocalTime."));
}
}
catch(QValueException qve)
{
throw (qve);
}
catch(Exception e)
{
throw (new QValueException("Value [" + value + "] could not be converted to a LocalTime.", e));
}
}
}

68
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java
View File

@ -22,12 +22,21 @@
package com.kingsrook.qqq.backend.core.actions.metadata;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
@ -47,9 +56,66 @@ class MetaDataActionTest
request.setSession(TestUtils.getMockSession());
MetaDataOutput result = new MetaDataAction().execute(request);
assertNotNull(result);
///////////////////////////////////
// assert against the tables map //
///////////////////////////////////
assertNotNull(result.getTables());
assertNotNull(result.getTables().get("person"));
assertEquals("person", result.getTables().get("person").getName());
assertEquals("Person", result.getTables().get("person").getLabel());
//////////////////////////////////////
// assert against the processes map //
//////////////////////////////////////
assertNotNull(result.getProcesses().get("greet"));
assertNotNull(result.getProcesses().get("greetInteractive"));
assertNotNull(result.getProcesses().get("etl.basic"));
assertNotNull(result.getProcesses().get("person.bulkInsert"));
assertNotNull(result.getProcesses().get("person.bulkEdit"));
assertNotNull(result.getProcesses().get("person.bulkDelete"));
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// assert against the apps map - which is appName to app - but not fully hierarchical - that's appTree //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
Map<String, QFrontendAppMetaData> apps = result.getApps();
assertNotNull(apps.get(TestUtils.APP_NAME_GREETINGS));
assertNotNull(apps.get(TestUtils.APP_NAME_PEOPLE));
assertNotNull(apps.get(TestUtils.APP_NAME_MISCELLANEOUS));
QFrontendAppMetaData peopleApp = apps.get(TestUtils.APP_NAME_PEOPLE);
assertThat(peopleApp.getChildren()).isNotEmpty();
Optional<AppTreeNode> greetingsAppUnderPeopleFromMapOptional = peopleApp.getChildren().stream()
.filter(e -> e.getName().equals(TestUtils.APP_NAME_GREETINGS)).findFirst();
assertThat(greetingsAppUnderPeopleFromMapOptional).isPresent();
//////////////////////////////////////////////////////////////////////////////
// we want to show that in the appMap (e.g., "apps"), that the apps are not //
// hierarchical - that is - that a sub-app doesn't list ITS children here. //
//////////////////////////////////////////////////////////////////////////////
assertThat(greetingsAppUnderPeopleFromMapOptional.get().getChildren()).isNullOrEmpty();
///////////////////////////////////////////////
// assert against the hierarchical apps tree //
///////////////////////////////////////////////
List<AppTreeNode> appTree = result.getAppTree();
Set<String> appNamesInTopOfTree = appTree.stream().map(AppTreeNode::getName).collect(Collectors.toSet());
assertThat(appNamesInTopOfTree).contains(TestUtils.APP_NAME_PEOPLE);
assertThat(appNamesInTopOfTree).contains(TestUtils.APP_NAME_MISCELLANEOUS);
assertThat(appNamesInTopOfTree).doesNotContain(TestUtils.APP_NAME_GREETINGS);
Optional<AppTreeNode> peopleAppOptional = appTree.stream()
.filter(e -> e.getName().equals(TestUtils.APP_NAME_PEOPLE)).findFirst();
assertThat(peopleAppOptional).isPresent();
assertThat(peopleAppOptional.get().getChildren()).isNotEmpty();
Optional<AppTreeNode> greetingsAppUnderPeopleFromTree = peopleAppOptional.get().getChildren().stream()
.filter(e -> e.getName().equals(TestUtils.APP_NAME_GREETINGS)).findFirst();
assertThat(greetingsAppUnderPeopleFromTree).isPresent();
/////////////////////////////////////////////////////////////////////////////////
// but here, when this app comes from the tree, then it DOES have its children //
/////////////////////////////////////////////////////////////////////////////////
assertThat(greetingsAppUnderPeopleFromTree.get().getChildren()).isNotEmpty();
}
}

3
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java
View File

@ -103,7 +103,8 @@ public class RunBackendStepActionTest
case STRING -> "ABC";
case INTEGER -> 42;
case DECIMAL -> new BigDecimal("47");
case DATE, DATE_TIME -> null;
case BOOLEAN -> true;
case DATE, TIME, DATE_TIME -> null;
case TEXT -> """
ABC
XYZ""";

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java
View File

@ -80,7 +80,7 @@ class ReportActionTest
public void testBigger() throws Exception
{
// int recordCount = 2_000_000; // to really stress locally, use this.
int recordCount = 200_000;
int recordCount = 50_000;
String filename = "/tmp/ReportActionTest.csv";
runReport(recordCount, filename, ReportFormat.CSV, false);

10
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java
View File

@ -25,8 +25,10 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@ -50,5 +52,13 @@ class QueryActionTest
request.setTableName("person");
QueryOutput result = new QueryAction().execute(request);
assertNotNull(result);
assertThat(result.getRecords()).isNotEmpty();
for(QRecord record : result.getRecords())
{
assertThat(record.getValues()).isNotEmpty();
assertThat(record.getDisplayValues()).isNotEmpty();
assertThat(record.getErrors()).isEmpty();
}
}
}

160
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java Normal file
View File

@ -0,0 +1,160 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.values;
import java.math.BigDecimal;
import java.util.Collections;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for QValueFormatter
*******************************************************************************/
class QValueFormatterTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFormatValue()
{
assertNull(QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), null));
assertEquals("1", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), 1));
assertEquals("1,000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), 1000));
assertEquals("1000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(null), 1000));
assertEquals("$1,000.00", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.CURRENCY), 1000));
assertEquals("1,000.00", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.DECIMAL2_COMMAS), 1000));
assertEquals("1000.00", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.DECIMAL2), 1000));
assertEquals("1", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), new BigDecimal("1")));
assertEquals("1,000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), new BigDecimal("1000")));
assertEquals("1000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.STRING), new BigDecimal("1000")));
assertEquals("1000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.STRING), 1000));
//////////////////////////////////////////////////
// this one flows through the exceptional cases //
//////////////////////////////////////////////////
assertEquals("1000.01", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), new BigDecimal("1000.01")));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFormatRecordLabel()
{
QTableMetaData table = new QTableMetaData().withRecordLabelFormat("%s %s").withRecordLabelFields(List.of("firstName", "lastName"));
assertEquals("Darin Kelkhoff", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff")));
assertEquals("Darin ", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("firstName", "Darin")));
assertEquals("Darin ", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("firstName", "Darin").withValue("lastName", null)));
table = new QTableMetaData().withRecordLabelFormat("%s " + DisplayFormat.CURRENCY).withRecordLabelFields(List.of("firstName", "price"));
assertEquals("Darin $10,000.00", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("firstName", "Darin").withValue("price", new BigDecimal(10000))));
table = new QTableMetaData().withRecordLabelFormat(DisplayFormat.DEFAULT).withRecordLabelFields(List.of("id"));
assertEquals("123456", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("id", "123456")));
///////////////////////////////////////////////////////
// exceptional flow: no recordLabelFormat specified //
///////////////////////////////////////////////////////
table = new QTableMetaData().withPrimaryKeyField("id");
assertEquals("42", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("id", 42)));
/////////////////////////////////////////////////
// exceptional flow: no fields for the format //
/////////////////////////////////////////////////
table = new QTableMetaData().withRecordLabelFormat("%s %s").withPrimaryKeyField("id");
assertEquals("128", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("id", 128)));
/////////////////////////////////////////////////////////
// exceptional flow: not enough fields for the format //
/////////////////////////////////////////////////////////
table = new QTableMetaData().withRecordLabelFormat("%s %s").withRecordLabelFields(List.of("a")).withPrimaryKeyField("id");
assertEquals("256", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("a", 47).withValue("id", 256)));
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// exceptional flow (kinda): too many fields for the format (just get the ones that are in the format) //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
table = new QTableMetaData().withRecordLabelFormat("%s %s").withRecordLabelFields(List.of("a", "b", "c")).withPrimaryKeyField("id");
assertEquals("47 48", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("a", 47).withValue("b", 48).withValue("c", 49).withValue("id", 256)));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSetDisplayValuesInRecords()
{
QTableMetaData table = new QTableMetaData()
.withRecordLabelFormat("%s %s")
.withRecordLabelFields(List.of("firstName", "lastName"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING))
.withField(new QFieldMetaData("lastName", QFieldType.STRING))
.withField(new QFieldMetaData("price", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY))
.withField(new QFieldMetaData("quantity", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS));
/////////////////////////////////////////////////////////////////
// first, make sure it doesn't crash with null or empty inputs //
/////////////////////////////////////////////////////////////////
QValueFormatter.setDisplayValuesInRecords(table, null);
QValueFormatter.setDisplayValuesInRecords(table, Collections.emptyList());
List<QRecord> records = List.of(
new QRecord()
.withValue("firstName", "Tim")
.withValue("lastName", "Chamberlain")
.withValue("price", new BigDecimal("3.50"))
.withValue("quantity", 1701),
new QRecord()
.withValue("firstName", "Tyler")
.withValue("lastName", "Samples")
.withValue("price", new BigDecimal("174999.99"))
.withValue("quantity", 47)
);
QValueFormatter.setDisplayValuesInRecords(table, records);
assertEquals("Tim Chamberlain", records.get(0).getRecordLabel());
assertEquals("$3.50", records.get(0).getDisplayValue("price"));
assertEquals("1,701", records.get(0).getDisplayValue("quantity"));
assertEquals("Tyler Samples", records.get(1).getRecordLabel());
assertEquals("$174,999.99", records.get(1).getDisplayValue("price"));
assertEquals("47", records.get(1).getDisplayValue("quantity"));
}
}

65
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java
View File

@ -22,8 +22,11 @@
package com.kingsrook.qqq.backend.core.instances;
import java.util.Collections;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
@ -45,7 +48,7 @@ class QInstanceEnricherTest
@Test
public void test_nullTableLabelComesFromName()
{
QInstance qInstance = TestUtils.defineInstance();
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData personTable = qInstance.getTable("person");
personTable.setLabel(null);
assertNull(personTable.getLabel());
@ -54,6 +57,7 @@ class QInstanceEnricherTest
}
/*******************************************************************************
** Test that a table missing a label and a name doesn't NPE, but just keeps
** the name & label both null.
@ -62,7 +66,7 @@ class QInstanceEnricherTest
@Test
public void test_nullNameGivesNullLabel()
{
QInstance qInstance = TestUtils.defineInstance();
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData personTable = qInstance.getTable("person");
personTable.setLabel(null);
personTable.setName(null);
@ -74,6 +78,7 @@ class QInstanceEnricherTest
}
/*******************************************************************************
** Test that a field missing a label gets the default label applied (name w/ UC-first)
**
@ -81,12 +86,64 @@ class QInstanceEnricherTest
@Test
public void test_nullFieldLabelComesFromName()
{
QInstance qInstance = TestUtils.defineInstance();
QFieldMetaData idField = qInstance.getTable("person").getField("id");
QInstance qInstance = TestUtils.defineInstance();
QFieldMetaData idField = qInstance.getTable("person").getField("id");
idField.setLabel(null);
assertNull(idField.getLabel());
new QInstanceEnricher().enrich(qInstance);
assertEquals("Id", idField.getLabel());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSetInferredFieldBackendNames()
{
QTableMetaData table = new QTableMetaData()
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("firstName", QFieldType.INTEGER))
.withField(new QFieldMetaData("nonstandard", QFieldType.INTEGER).withBackendName("whateverImNon_standard"));
QInstanceEnricher.setInferredFieldBackendNames(table);
assertEquals("id", table.getField("id").getBackendName());
assertEquals("first_name", table.getField("firstName").getBackendName());
assertEquals("whateverImNon_standard", table.getField("nonstandard").getBackendName());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSetInferredFieldBackendNamesEdgeCases()
{
///////////////////////////////////////////////////////////////
// make sure none of these cases throw (but all should warn) //
///////////////////////////////////////////////////////////////
QInstanceEnricher.setInferredFieldBackendNames(null);
QInstanceEnricher.setInferredFieldBackendNames(new QTableMetaData());
QInstanceEnricher.setInferredFieldBackendNames(new QTableMetaData().withFields(Collections.emptyMap()));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testInferBackendName()
{
assertEquals("id", QInstanceEnricher.inferBackendName("id"));
assertEquals("word_another_word_more_words", QInstanceEnricher.inferBackendName("wordAnotherWordMoreWords"));
assertEquals("l_ul_ul_ul", QInstanceEnricher.inferBackendName("lUlUlUl"));
assertEquals("starts_upper", QInstanceEnricher.inferBackendName("StartsUpper"));
assertEquals("tla_first", QInstanceEnricher.inferBackendName("TLAFirst"));
assertEquals("word_then_tla_in_middle", QInstanceEnricher.inferBackendName("wordThenTLAInMiddle"));
assertEquals("end_with_tla", QInstanceEnricher.inferBackendName("endWithTLA"));
assertEquals("tla_and_another_tla", QInstanceEnricher.inferBackendName("TLAAndAnotherTLA"));
}
}

410
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
View File

@ -22,13 +22,24 @@
package com.kingsrook.qqq.backend.core.instances;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@ -51,6 +62,21 @@ class QInstanceValidatorTest
/*******************************************************************************
** make sure we don't re-validate if already validated
**
*******************************************************************************/
@Test
public void test_doNotReValidate() throws QInstanceValidationException
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.setHasBeenValidated(new QInstanceValidationKey());
qInstance.setBackends(null);
new QInstanceValidator().validate(qInstance);
}
/*******************************************************************************
** Test an instance with null backends - should throw.
**
@ -58,17 +84,8 @@ class QInstanceValidatorTest
@Test
public void test_validateNullBackends()
{
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.setBackends(null);
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("At least 1 backend must be defined", e);
}
assertValidationFailureReasons((qInstance) -> qInstance.setBackends(null),
"At least 1 backend must be defined");
}
@ -80,17 +97,8 @@ class QInstanceValidatorTest
@Test
public void test_validateEmptyBackends()
{
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.setBackends(new HashMap<>());
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("At least 1 backend must be defined", e);
}
assertValidationFailureReasons((qInstance) -> qInstance.setBackends(new HashMap<>()),
"At least 1 backend must be defined");
}
@ -102,17 +110,12 @@ class QInstanceValidatorTest
@Test
public void test_validateNullTables()
{
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.setTables(null);
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("At least 1 table must be defined", e);
}
assertValidationFailureReasons((qInstance) ->
{
qInstance.setTables(null);
qInstance.setProcesses(null);
},
"At least 1 table must be defined");
}
@ -124,17 +127,12 @@ class QInstanceValidatorTest
@Test
public void test_validateEmptyTables()
{
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.setTables(new HashMap<>());
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("At least 1 table must be defined", e);
}
assertValidationFailureReasons((qInstance) ->
{
qInstance.setTables(new HashMap<>());
qInstance.setProcesses(new HashMap<>());
},
"At least 1 table must be defined");
}
@ -147,19 +145,15 @@ class QInstanceValidatorTest
@Test
public void test_validateInconsistentNames()
{
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.getTable("person").setName("notPerson");
qInstance.getBackend("default").setName("notDefault");
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("Inconsistent naming for table", e);
assertReason("Inconsistent naming for backend", e);
}
assertValidationFailureReasonsAllowingExtraReasons((qInstance) ->
{
qInstance.getTable("person").setName("notPerson");
qInstance.getBackend("default").setName("notDefault");
qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setName("notGreetPeople");
},
"Inconsistent naming for table",
"Inconsistent naming for backend",
"Inconsistent naming for process");
}
@ -171,17 +165,8 @@ class QInstanceValidatorTest
@Test
public void test_validateTableWithoutBackend()
{
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.getTable("person").setBackendName(null);
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("Missing backend name for table", e);
}
assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setBackendName(null),
"Missing backend name for table");
}
@ -193,17 +178,53 @@ class QInstanceValidatorTest
@Test
public void test_validateTableWithMissingBackend()
{
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.getTable("person").setBackendName("notARealBackend");
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("Unrecognized backend", e);
}
assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setBackendName("notARealBackend"),
"Unrecognized backend");
}
/*******************************************************************************
** Test that if a process specifies a table that doesn't exist, that it fails.
**
*******************************************************************************/
@Test
public void test_validateProcessWithMissingTable()
{
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setTableName("notATableName"),
"Unrecognized table");
}
/*******************************************************************************
** Test that a process with no steps fails
**
*******************************************************************************/
@Test
public void test_validateProcessWithNoSteps()
{
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setStepList(Collections.emptyList()),
"At least 1 step");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setStepList(null),
"At least 1 step");
}
/*******************************************************************************
** Test that a process step with an empty string name fails
**
*******************************************************************************/
@Test
public void test_validateProcessStepWithEmptyName()
{
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName(""),
"Missing name for a step");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null),
"Missing name for a step");
}
@ -215,29 +236,11 @@ class QInstanceValidatorTest
@Test
public void test_validateTableWithNoFields()
{
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.getTable("person").setFields(null);
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("At least 1 field", e);
}
assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(null),
"At least 1 field");
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.getTable("person").setFields(new HashMap<>());
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("At least 1 field", e);
}
assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(new HashMap<>()),
"At least 1 field");
}
@ -248,17 +251,210 @@ class QInstanceValidatorTest
*******************************************************************************/
@Test
public void test_validateFieldWithMissingPossibleValueSource()
{
assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").getField("homeState").setPossibleValueSourceName("not a real possible value source"),
"Unrecognized possibleValueSourceName");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testChildrenWithBadParentAppName()
{
String[] reasons = new String[] { "Unrecognized parent app", "does not have its parent app properly set" };
assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON).setParentAppName("notAnApp"), reasons);
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setParentAppName("notAnApp"), reasons);
assertValidationFailureReasons((qInstance) -> qInstance.getApp(TestUtils.APP_NAME_GREETINGS).setParentAppName("notAnApp"), reasons);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAppCircularReferences()
{
assertValidationFailureReasonsAllowingExtraReasons((qInstance) ->
{
QAppMetaData miscApp = qInstance.getApp(TestUtils.APP_NAME_MISCELLANEOUS);
QAppMetaData greetingsApp = qInstance.getApp(TestUtils.APP_NAME_GREETINGS);
miscApp.withChild(greetingsApp);
greetingsApp.withChild(miscApp);
}, "Circular app reference");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFieldSectionsMissingName()
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withSection(new QFieldSection(null, "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "Missing a name");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFieldSectionsMissingLabel()
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withSection(new QFieldSection("section1", null, new QIcon("person"), Tier.T1, List.of("id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "Missing a label");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFieldSectionsNoFields()
{
QTableMetaData table1 = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of()))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "section1 does not have any fields", "field id is not listed in any field sections");
QTableMetaData table2 = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, null))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table2), "section1 does not have any fields", "field id is not listed in any field sections");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFieldSectionsUnrecognizedFieldName()
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "od")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "not a field on this table");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFieldSectionsDuplicatedFieldName()
{
QTableMetaData table1 = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "more than once");
QTableMetaData table2 = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T2, List.of("id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table2), "more than once");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFieldNotInAnySections()
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("name", QFieldType.STRING));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "not listed in any field sections");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFieldSectionsMultipleTier1()
{
QTableMetaData table = new QTableMetaData().withName("test")
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
.withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id")))
.withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T1, List.of("name")))
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("name", QFieldType.STRING));
assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "more than 1 section listed as Tier 1");
}
/*******************************************************************************
** Run a little setup code on a qInstance; then validate it, and assert that it
** failed validation with reasons that match the supplied vararg-reasons (but allow
** more reasons - e.g., helpful when one thing we're testing causes other errors).
*******************************************************************************/
private void assertValidationFailureReasonsAllowingExtraReasons(Consumer<QInstance> setup, String... reasons)
{
assertValidationFailureReasons(setup, true, reasons);
}
/*******************************************************************************
** Run a little setup code on a qInstance; then validate it, and assert that it
** failed validation with reasons that match the supplied vararg-reasons (and
** require that exact # of reasons).
*******************************************************************************/
private void assertValidationFailureReasons(Consumer<QInstance> setup, String... reasons)
{
assertValidationFailureReasons(setup, false, reasons);
}
/*******************************************************************************
** Implementation for the overloads of this name.
*******************************************************************************/
private void assertValidationFailureReasons(Consumer<QInstance> setup, boolean allowExtraReasons, String... reasons)
{
try
{
QInstance qInstance = TestUtils.defineInstance();
qInstance.getTable("person").getField("homeState").setPossibleValueSourceName("not a real possible value source");
setup.accept(qInstance);
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
assertReason("Unrecognized possibleValueSourceName", e);
if(!allowExtraReasons)
{
assertEquals(reasons.length, e.getReasons().size(), "Expected number of validation failure reasons\nExpected: " + String.join(",", reasons) + "\nActual: " + e.getReasons());
}
for(String reason : reasons)
{
assertReason(reason, e);
}
}
}
@ -271,7 +467,9 @@ class QInstanceValidatorTest
*******************************************************************************/
private void assertReason(String reason, QInstanceValidationException e)
{
assertNotNull(e.getReasons());
assertTrue(e.getReasons().stream().anyMatch(s -> s.contains(reason)));
assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)");
assertThat(e.getReasons())
.withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason)
.anyMatch(s -> s.contains(reason));
}
}

66
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java
View File

@ -26,6 +26,7 @@ import java.math.BigDecimal;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.testentities.Item;
import com.kingsrook.qqq.backend.core.model.data.testentities.ItemWithPrimitives;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -162,15 +163,76 @@ class QRecordEntityTest
*******************************************************************************/
@SuppressWarnings("ResultOfMethodCallIgnored")
@Test
void testQTableConstructionFromEntity() throws QException
void testQTableConstructionFromEntityGetterReferences() throws QException
{
QTableMetaData qTableMetaData = new QTableMetaData()
.withField(new QFieldMetaData(Item::getSku))
.withField(new QFieldMetaData(Item::getDescription))
.withField(new QFieldMetaData(Item::getQuantity));
.withField(new QFieldMetaData(Item::getQuantity))
.withField(new QFieldMetaData(Item::getFeatured))
.withField(new QFieldMetaData(Item::getPrice));
assertEquals(QFieldType.STRING, qTableMetaData.getField("sku").getType());
assertEquals(QFieldType.INTEGER, qTableMetaData.getField("quantity").getType());
///////////////////////////////////////////////////////////////
// assert about attributes that came from @QField annotation //
///////////////////////////////////////////////////////////////
assertEquals("SKU", qTableMetaData.getField("sku").getLabel());
assertEquals(DisplayFormat.COMMAS, qTableMetaData.getField("quantity").getDisplayFormat());
assertTrue(qTableMetaData.getField("sku").getIsRequired());
assertFalse(qTableMetaData.getField("quantity").getIsEditable());
assertEquals("is_featured", qTableMetaData.getField("featured").getBackendName());
//////////////////////////////////////////////////////////////////////////
// assert about attributes that weren't specified in @QField annotation //
//////////////////////////////////////////////////////////////////////////
assertTrue(qTableMetaData.getField("sku").getIsEditable());
assertFalse(qTableMetaData.getField("quantity").getIsRequired());
assertNull(qTableMetaData.getField("sku").getBackendName());
/////////////////////////////////////////////////////////////////////
// assert about attributes for fields without a @QField annotation //
/////////////////////////////////////////////////////////////////////
assertTrue(qTableMetaData.getField("price").getIsEditable());
assertFalse(qTableMetaData.getField("price").getIsRequired());
assertNull(qTableMetaData.getField("price").getBackendName());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testQTableConstructionFromEntity() throws QException
{
QTableMetaData qTableMetaData = new QTableMetaData()
.withFieldsFromEntity(Item.class);
assertEquals(QFieldType.STRING, qTableMetaData.getField("sku").getType());
assertEquals(QFieldType.INTEGER, qTableMetaData.getField("quantity").getType());
///////////////////////////////////////////////////////////////
// assert about attributes that came from @QField annotation //
///////////////////////////////////////////////////////////////
assertTrue(qTableMetaData.getField("sku").getIsRequired());
assertFalse(qTableMetaData.getField("quantity").getIsEditable());
assertEquals("is_featured", qTableMetaData.getField("featured").getBackendName());
//////////////////////////////////////////////////////////////////////////
// assert about attributes that weren't specified in @QField annotation //
//////////////////////////////////////////////////////////////////////////
assertTrue(qTableMetaData.getField("sku").getIsEditable());
assertFalse(qTableMetaData.getField("quantity").getIsRequired());
assertNull(qTableMetaData.getField("sku").getBackendName());
/////////////////////////////////////////////////////////////////////
// assert about attributes for fields without a @QField annotation //
/////////////////////////////////////////////////////////////////////
assertTrue(qTableMetaData.getField("price").getIsEditable());
assertFalse(qTableMetaData.getField("price").getIsRequired());
assertNull(qTableMetaData.getField("price").getBackendName());
}

18
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java
View File

@ -23,7 +23,9 @@ package com.kingsrook.qqq.backend.core.model.data.testentities;
import java.math.BigDecimal;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
/*******************************************************************************
@ -31,11 +33,19 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
*******************************************************************************/
public class Item extends QRecordEntity
{
private String sku;
private String description;
private Integer quantity;
@QField(isRequired = true, label = "SKU")
private String sku;
@QField()
private String description;
@QField(isEditable = false, displayFormat = DisplayFormat.COMMAS)
private Integer quantity;
private BigDecimal price;
private Boolean featured;
@QField(backendName = "is_featured")
private Boolean featured;

76
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.modules.authentication;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
@ -31,7 +32,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.Auth0AuthenticationMetaData;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.AUTH0_ID_TOKEN_KEY;
@ -40,7 +40,8 @@ import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0Authent
import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.INVALID_TOKEN_ERROR;
import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.TOKEN_NOT_PROVIDED_ERROR;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@ -49,9 +50,9 @@ import static org.junit.jupiter.api.Assertions.fail;
*******************************************************************************/
public class Auth0AuthenticationModuleTest
{
private static final String VALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA";
private static final String INVALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA-thismakesinvalid";
private static final String EXPIRED_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE4VDIxOjM4OjE1LjM4NloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MTgwNDc3LCJleHAiOjE2NTgyMTY0NzcsIm5vbmNlIjoiVkZkQlYzWmplR2hvY1cwMk9WZEtabHBLU0c1K1ZXbElhMEV3VkZaeFpVdEJVMDErZUZaT1RtMTNiZz09In0.fU7EwUgNrupOPz_PX_aQKON2xG1-LWD85xVo1Bn41WNEek-iMyJoch8l6NUihi7Bou14BoOfeWIG_sMqsLHqI2Pk7el7l1kigsjURx0wpiXadBt8piMxdIlxdToZEMuZCBzg7eJvXh4sM8tlV5cm0gPa6FT9Ih3VGJajNlXi5BcYS_JRpIvFvHn8-Bxj4KiAlZ5XPPkopjnDgP8kFfc4cMn_nxDkqWYlhj-5TaGW2xCLC9Qr_9UNxX0fm-CkKjYs3Z5ezbiXNkc-bxrCYvxeBeDPf8-T3EqrxCRVqCZSJ85BHdOc_E7UZC_g8bNj0umoplGwlCbzO4XIuOO-KlIaOg";
private static final String VALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA";
private static final String INVALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA-thismakesinvalid";
private static final String EXPIRED_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE4VDIxOjM4OjE1LjM4NloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MTgwNDc3LCJleHAiOjE2NTgyMTY0NzcsIm5vbmNlIjoiVkZkQlYzWmplR2hvY1cwMk9WZEtabHBLU0c1K1ZXbElhMEV3VkZaeFpVdEJVMDErZUZaT1RtMTNiZz09In0.fU7EwUgNrupOPz_PX_aQKON2xG1-LWD85xVo1Bn41WNEek-iMyJoch8l6NUihi7Bou14BoOfeWIG_sMqsLHqI2Pk7el7l1kigsjURx0wpiXadBt8piMxdIlxdToZEMuZCBzg7eJvXh4sM8tlV5cm0gPa6FT9Ih3VGJajNlXi5BcYS_JRpIvFvHn8-Bxj4KiAlZ5XPPkopjnDgP8kFfc4cMn_nxDkqWYlhj-5TaGW2xCLC9Qr_9UNxX0fm-CkKjYs3Z5ezbiXNkc-bxrCYvxeBeDPf8-T3EqrxCRVqCZSJ85BHdOc_E7UZC_g8bNj0umoplGwlCbzO4XIuOO-KlIaOg";
private static final String UNDECODABLE_TOKEN = "UNDECODABLE";
public static final String AUTH0_BASE_URL = "https://kingsrook.us.auth0.com/";
@ -59,32 +60,66 @@ public class Auth0AuthenticationModuleTest
/*******************************************************************************
** Test a valid token where 'now' is set to a time that would be valid for it
** Test a token where last-checked is set to a time that would not require it to be
** re-checked, so it'll show as valid no matter what the token is.
**
*******************************************************************************/
@Test
public void testLastTimeChecked() throws QAuthenticationException
public void testLastTimeCheckedJustUnderThreshold()
{
//////////////////////////////////////////////////////////
// Tuesday, July 19, 2022 12:40:27.299 PM GMT-05:00 DST //
//////////////////////////////////////////////////////////
Instant now = Instant.now();
Instant underThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS - 60, ChronoUnit.SECONDS);
assertTrue(testLastTimeChecked(underThreshold, INVALID_TOKEN), "A session checked under threshold should be valid");
}
/////////////////////////////////////////////////////////
// put the 'now' from the past into the state provider //
/////////////////////////////////////////////////////////
StateProviderInterface spi = InMemoryStateProvider.getInstance();
Auth0AuthenticationModule.Auth0StateKey key = new Auth0AuthenticationModule.Auth0StateKey(VALID_TOKEN);
spi.put(key, now);
/*******************************************************************************
** Test a token where last-checked is set to a time that would require it to be
** re-checked, so it'll show as invalid.
**
*******************************************************************************/
@Test
public void testLastTimeCheckedJustOverThreshold()
{
Instant overThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS + 60, ChronoUnit.SECONDS);
assertFalse(testLastTimeChecked(overThreshold, INVALID_TOKEN), "A session checked over threshold should be re-validated, and in this case, not be valid.");
}
/*******************************************************************************
** Test a token where last-checked is past the threshold, so it'll get re-checked,
** and will fail.
**
*******************************************************************************/
@Test
public void testLastTimeCheckedOverThresholdAndUndecodable()
{
Instant overThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS + 60, ChronoUnit.SECONDS);
assertFalse(testLastTimeChecked(overThreshold, UNDECODABLE_TOKEN), "A session checked over threshold should be re-validated, and in this case, not be valid.");
}
/*******************************************************************************
**
*******************************************************************************/
private boolean testLastTimeChecked(Instant lastTimeChecked, String token)
{
/////////////////////////////////////////////////////////////
// put the input last-time-checked into the state provider //
/////////////////////////////////////////////////////////////
Auth0AuthenticationModule.Auth0StateKey key = new Auth0AuthenticationModule.Auth0StateKey(token);
InMemoryStateProvider.getInstance().put(key, lastTimeChecked);
//////////////////////
// build up session //
//////////////////////
QSession session = new QSession();
session.setIdReference(VALID_TOKEN);
session.setIdReference(token);
Auth0AuthenticationModule auth0AuthenticationModule = new Auth0AuthenticationModule();
assertEquals(true, auth0AuthenticationModule.isSessionValid(session), "Session should return as still valid.");
return (auth0AuthenticationModule.isSessionValid(getQInstance(), session));
}
@ -114,7 +149,7 @@ public class Auth0AuthenticationModuleTest
/*******************************************************************************
** Test failure case, token cant be decoded
** Test failure case, token can't be decoded
**
*******************************************************************************/
@Test
@ -220,4 +255,5 @@ public class Auth0AuthenticationModuleTest
qInstance.setAuthentication(authenticationMetaData);
return (qInstance);
}
}

4
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java
View File

@ -49,8 +49,8 @@ public class FullyAnonymousAuthenticationModuleTest
assertNotNull(session.getIdReference(), "Session id ref should not be null");
assertNotNull(session.getUser(), "Session User should not be null");
assertNotNull(session.getUser().getIdReference(), "Session User id ref should not be null");
assertTrue(fullyAnonymousAuthenticationModule.isSessionValid(session), "Any session should be valid");
assertFalse(fullyAnonymousAuthenticationModule.isSessionValid(null), "null should be not valid");
assertTrue(fullyAnonymousAuthenticationModule.isSessionValid(null, session), "Any session should be valid");
assertFalse(fullyAnonymousAuthenticationModule.isSessionValid(null, null), "null should be not valid");
}
}

94
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java Normal file
View File

@ -0,0 +1,94 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLProcess
*******************************************************************************/
class StreamedETLProcessTest
{
/*******************************************************************************
** Simplest happy path
*******************************************************************************/
@Test
public void test() throws QException
{
RunProcessInput request = new RunProcessInput(TestUtils.defineInstance());
request.setSession(TestUtils.getMockSession());
request.setProcessName(StreamedETLProcess.PROCESS_NAME);
request.addValue(StreamedETLProcess.FIELD_SOURCE_TABLE, TestUtils.defineTablePerson().getName());
request.addValue(StreamedETLProcess.FIELD_DESTINATION_TABLE, TestUtils.definePersonFileTable().getName());
request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, "");
RunProcessOutput result = new RunProcessAction().execute(request);
assertNotNull(result);
///////////////////////////////////////////////////////////////////////
// since this is streamed, assert there are no records in the output //
///////////////////////////////////////////////////////////////////////
assertTrue(result.getRecords().isEmpty());
assertTrue(result.getException().isEmpty());
}
/*******************************************************************************
** Basic example of doing a mapping transformation
*******************************************************************************/
@Test
public void testMappingTransformation() throws QException
{
RunProcessInput request = new RunProcessInput(TestUtils.defineInstance());
request.setSession(TestUtils.getMockSession());
request.setProcessName(StreamedETLProcess.PROCESS_NAME);
request.addValue(StreamedETLProcess.FIELD_SOURCE_TABLE, TestUtils.definePersonFileTable().getName());
request.addValue(StreamedETLProcess.FIELD_DESTINATION_TABLE, TestUtils.defineTableIdAndNameOnly().getName());
///////////////////////////////////////////////////////////////////////////////////////
// define our mapping from destination-table field names to source-table field names //
///////////////////////////////////////////////////////////////////////////////////////
QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping().withMapping("name", "firstName");
request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping));
RunProcessOutput result = new RunProcessAction().execute(request);
assertNotNull(result);
///////////////////////////////////////////////////////////////////////
// since this is streamed, assert there are no records in the output //
///////////////////////////////////////////////////////////////////////
assertTrue(result.getRecords().isEmpty());
assertTrue(result.getException().isEmpty());
}
}

84
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -32,16 +32,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
@ -50,10 +48,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
/*******************************************************************************
@ -62,9 +64,19 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicE
*******************************************************************************/
public class TestUtils
{
public static String DEFAULT_BACKEND_NAME = "default";
public static String PROCESS_NAME_GREET_PEOPLE = "greet";
public static String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive";
public static final String DEFAULT_BACKEND_NAME = "default";
public static final String APP_NAME_GREETINGS = "greetingsApp";
public static final String APP_NAME_PEOPLE = "peopleApp";
public static final String APP_NAME_MISCELLANEOUS = "miscellaneous";
public static final String TABLE_NAME_PERSON = "person";
public static final String PROCESS_NAME_GREET_PEOPLE = "greet";
public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive";
public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE = "addToPeoplesAge";
public static final String TABLE_NAME_PERSON_FILE = "personFile";
public static final String TABLE_NAME_ID_AND_NAME_ONLY = "idAndNameOnly";
@ -77,14 +89,20 @@ public class TestUtils
QInstance qInstance = new QInstance();
qInstance.setAuthentication(defineAuthentication());
qInstance.addBackend(defineBackend());
qInstance.addTable(defineTablePerson());
qInstance.addTable(definePersonFileTable());
qInstance.addTable(defineTableIdAndNameOnly());
qInstance.addPossibleValueSource(defineStatesPossibleValueSource());
qInstance.addProcess(defineProcessGreetPeople());
qInstance.addProcess(defineProcessGreetPeopleInteractive());
qInstance.addProcess(defineProcessAddToPeoplesAge());
qInstance.addProcess(new BasicETLProcess().defineProcessMetaData());
qInstance.addProcess(new StreamedETLProcess().defineProcessMetaData());
defineApps(qInstance);
System.out.println(new QInstanceAdapter().qInstanceToJson(qInstance));
@ -93,6 +111,30 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
private static void defineApps(QInstance qInstance)
{
qInstance.addApp(new QAppMetaData()
.withName(APP_NAME_GREETINGS)
.withChild(qInstance.getProcess(PROCESS_NAME_GREET_PEOPLE))
.withChild(qInstance.getProcess(PROCESS_NAME_GREET_PEOPLE_INTERACTIVE)));
qInstance.addApp(new QAppMetaData()
.withName(APP_NAME_PEOPLE)
.withChild(qInstance.getTable(TABLE_NAME_PERSON))
.withChild(qInstance.getTable(TABLE_NAME_PERSON_FILE))
.withChild(qInstance.getApp(APP_NAME_GREETINGS)));
qInstance.addApp(new QAppMetaData()
.withName(APP_NAME_MISCELLANEOUS)
.withChild(qInstance.getTable(TABLE_NAME_ID_AND_NAME_ONLY))
.withChild(qInstance.getProcess(BasicETLProcess.PROCESS_NAME)));
}
/*******************************************************************************
** Define the "states" possible value source used in standard tests
**
@ -138,7 +180,7 @@ public class TestUtils
public static QTableMetaData defineTablePerson()
{
return new QTableMetaData()
.withName("person")
.withName(TABLE_NAME_PERSON)
.withLabel("Person")
.withBackendName(DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
@ -160,7 +202,7 @@ public class TestUtils
public static QTableMetaData definePersonFileTable()
{
return (new QTableMetaData()
.withName("personFile")
.withName(TABLE_NAME_PERSON_FILE)
.withLabel("Person File")
.withBackendName(DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
@ -175,7 +217,7 @@ public class TestUtils
public static QTableMetaData defineTableIdAndNameOnly()
{
return new QTableMetaData()
.withName("idAndNameOnly")
.withName(TABLE_NAME_ID_AND_NAME_ONLY)
.withLabel("Id and Name Only")
.withBackendName(DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
@ -192,7 +234,7 @@ public class TestUtils
{
return new QProcessMetaData()
.withName(PROCESS_NAME_GREET_PEOPLE)
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addStep(new QBackendStepMetaData()
.withName("prepare")
.withCode(new QCodeReference()
@ -200,14 +242,14 @@ public class TestUtils
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
.withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON))
.withFieldList(List.of(
new QFieldMetaData("greetingPrefix", QFieldType.STRING),
new QFieldMetaData("greetingSuffix", QFieldType.STRING)
)))
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
@ -223,7 +265,7 @@ public class TestUtils
{
return new QProcessMetaData()
.withName(PROCESS_NAME_GREET_PEOPLE_INTERACTIVE)
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addStep(new QFrontendStepMetaData()
.withName("setup")
@ -238,14 +280,14 @@ public class TestUtils
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
.withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON))
.withFieldList(List.of(
new QFieldMetaData("greetingPrefix", QFieldType.STRING),
new QFieldMetaData("greetingSuffix", QFieldType.STRING)
)))
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
@ -270,8 +312,8 @@ public class TestUtils
private static QProcessMetaData defineProcessAddToPeoplesAge()
{
return new QProcessMetaData()
.withName("addToPeoplesAge")
.withTableName("person")
.withName(PROCESS_NAME_ADD_TO_PEOPLES_AGE)
.withTableName(TABLE_NAME_PERSON)
.addStep(new QBackendStepMetaData()
.withName("getAgeStatistics")
.withCode(new QCodeReference()
@ -279,10 +321,10 @@ public class TestUtils
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName("person")))
.withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON)))
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("age", QFieldType.INTEGER)))
.withFieldList(List.of(
new QFieldMetaData("minAge", QFieldType.INTEGER),
@ -297,7 +339,7 @@ public class TestUtils
.withFieldList(List.of(new QFieldMetaData("yearsToAdd", QFieldType.INTEGER))))
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("newAge", QFieldType.INTEGER)))));
}

113
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java
View File

@ -24,9 +24,21 @@ package com.kingsrook.qqq.backend.core.utils;
import java.math.BigDecimal;
import java.math.MathContext;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.Month;
import java.util.Calendar;
import java.util.GregorianCalendar;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
@ -132,7 +144,9 @@ class ValueUtilsTest
assertEquals(new BigDecimal("1"), ValueUtils.getValueAsBigDecimal(1.0F));
assertEquals(new BigDecimal("1"), ValueUtils.getValueAsBigDecimal(1.0D));
assertEquals(new BigDecimal("1000000000000"), ValueUtils.getValueAsBigDecimal(1_000_000_000_000L));
//noinspection ConstantConditions
assertEquals(0, new BigDecimal("1.1").compareTo(ValueUtils.getValueAsBigDecimal(1.1F).round(MathContext.DECIMAL32)));
//noinspection ConstantConditions
assertEquals(0, new BigDecimal("1.1").compareTo(ValueUtils.getValueAsBigDecimal(1.1D).round(MathContext.DECIMAL64)));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsBigDecimal("a"));
@ -140,4 +154,101 @@ class ValueUtilsTest
assertThrows(QValueException.class, () -> ValueUtils.getValueAsBigDecimal(new Object()));
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("deprecation")
@Test
void testGetValueAsLocalDate() throws QValueException
{
LocalDate expected = LocalDate.of(1980, Month.MAY, 31);
assertNull(ValueUtils.getValueAsLocalDate(null));
assertNull(ValueUtils.getValueAsLocalDate(""));
assertNull(ValueUtils.getValueAsLocalDate(" "));
assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDate.of(1980, 5, 31)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.sql.Date(80, 4, 31)));
//noinspection MagicConstant
assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, 4, 31)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 12, 0)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 4, 0)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 22, 0)));
//noinspection MagicConstant
assertEquals(expected, ValueUtils.getValueAsLocalDate(new GregorianCalendar(1980, 4, 31)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(new GregorianCalendar(1980, Calendar.MAY, 31)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 12, 0)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 4, 0)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 22, 0)));
assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, Month.MAY, 31, 12, 0)));
assertEquals(expected, ValueUtils.getValueAsLocalDate("1980-05-31"));
assertEquals(expected, ValueUtils.getValueAsLocalDate("05/31/1980"));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate("a"));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate("a,b"));
assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate("1980/05/31")).getMessage()).contains("parse");
assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate(new Object())).getMessage()).contains("Unsupported class");
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("deprecation")
@Test
void testGetValueAsInstant() throws QValueException
{
Instant expected = Instant.parse("1980-05-31T12:30:00Z");
assertNull(ValueUtils.getValueAsInstant(null));
assertNull(ValueUtils.getValueAsInstant(""));
assertNull(ValueUtils.getValueAsInstant(" "));
assertEquals(expected, ValueUtils.getValueAsInstant(expected));
assertEquals(expected, ValueUtils.getValueAsInstant("1980-05-31T12:30:00Z"));
////////////////////////////
// todo - time zone logic //
////////////////////////////
// //noinspection MagicConstant
// assertEquals(expected, ValueUtils.getValueAsInstant(new java.util.Date(80, 4, 31, 7, 30)));
// //noinspection MagicConstant
// assertEquals(expected, ValueUtils.getValueAsInstant(new GregorianCalendar(1980, 4, 31)));
// assertEquals(expected, ValueUtils.getValueAsInstant(new GregorianCalendar(1980, Calendar.MAY, 31)));
// // assertEquals(expected, ValueUtils.getValueAsInstant(InstantTime.of(1980, 5, 31, 12, 0)));
// // assertEquals(expected, ValueUtils.getValueAsInstant(InstantTime.of(1980, 5, 31, 4, 0)));
// // assertEquals(expected, ValueUtils.getValueAsInstant(InstantTime.of(1980, 5, 31, 22, 0)));
// // assertEquals(expected, ValueUtils.getValueAsInstant(InstantTime.of(1980, Month.MAY, 31, 12, 0)));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant(new java.sql.Date(80, 4, 31)));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a"));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a,b"));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("1980/05/31"));
assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant(new Object())).getMessage()).contains("Unsupported class");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetValueAsLocalTime() throws QValueException
{
assertNull(ValueUtils.getValueAsInstant(null));
assertNull(ValueUtils.getValueAsInstant(""));
assertNull(ValueUtils.getValueAsInstant(" "));
assertEquals(LocalTime.of(10, 42), ValueUtils.getValueAsLocalTime(LocalTime.of(10, 42)));
assertEquals(LocalTime.of(10, 42, 59), ValueUtils.getValueAsLocalTime(LocalTime.of(10, 42, 59)));
assertEquals(LocalTime.of(10, 42), ValueUtils.getValueAsLocalTime("10:42"));
assertEquals(LocalTime.of(10, 42, 59), ValueUtils.getValueAsLocalTime("10:42:59"));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a"));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a,b"));
assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("1980/05/31"));
assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant(new Object())).getMessage()).contains("Unsupported class");
}
}

30
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -201,10 +201,16 @@ public abstract class AbstractBaseFilesystemAction<FILE>
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
if(queryInput.getRecordPipe() != null)
{
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> addBackendDetailsToRecord(record, file)));
}
else
{
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
}
break;
}
case JSON:
@ -212,6 +218,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
// todo - pipe support!!
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
@ -241,10 +248,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/
protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, FILE file)
{
recordsInFile.forEach(record ->
{
record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file));
});
recordsInFile.forEach(r -> addBackendDetailsToRecord(r, file));
}
/*******************************************************************************
** Add backend details to a record about the file that it is in.
*******************************************************************************/
protected void addBackendDetailsToRecord(QRecord record, FILE file)
{
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file));
}

2
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java
View File

@ -94,11 +94,13 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep
String moveOrDelete = runBackendStepInput.getValueString(FIELD_MOVE_OR_DELETE);
if(VALUE_DELETE.equals(moveOrDelete))
{
LOG.info("Deleting ETL source file: " + sourceFile);
actionBase.deleteFile(runBackendStepInput.getInstance(), table, sourceFile);
}
else if(VALUE_MOVE.equals(moveOrDelete))
{
String destinationForMoves = runBackendStepInput.getValueString(FIELD_DESTINATION_FOR_MOVES);
LOG.info("Moving ETL source file: " + sourceFile + " to " + destinationForMoves);
if(!StringUtils.hasContent(destinationForMoves))
{
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));

75
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStep.java Normal file
View File

@ -0,0 +1,75 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLBackendStep;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic.BasicETLCollectSourceFileNamesStep;
/*******************************************************************************
** Extension to the base StreamedETLBackendStep, unique for where the source
** table is a filesystem, where we want/need to collect the filenames that were
** processed in the Extract step, so they can be passed into the cleanup step.
**
** Similar in purpose to the BasicETLCollectSourceFileNamesStep - only in this
** case, due to the streaming behavior of the StreamedETLProcess, we can't really
** inject this code as a separate backend step - so instead we extend that step,
** and override its postTransform method to intercept the records & file names.
*******************************************************************************/
public class StreamedETLFilesystemBackendStep extends StreamedETLBackendStep
{
/*******************************************************************************
**
*******************************************************************************/
@Override
protected void preTransform(List<QRecord> qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
Set<String> sourceFiles = qRecords.stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet());
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// expect that we'll be called on multiple "pages" of records as they run through the pipe. //
// each time we're called, we need to: //
// - get the unique file paths in this list of records //
// - if we previously set the list of file names in the output, then split that value up and add those names to the set we see now //
// - set the list of name (joined by commas) in the output //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
String existingListOfFileNames = runBackendStepOutput.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS);
if(existingListOfFileNames != null)
{
sourceFiles.addAll(List.of(existingListOfFileNames.split(",")));
}
runBackendStepOutput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
}
}

4
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java
View File

@ -137,7 +137,7 @@ public class S3Utils
////////////////////////////////////////////////////////////////////////////////
if(key.endsWith("/"))
{
LOG.debug("Skipping file [{}] because it is a folder", key);
// LOG.debug("Skipping file [{}] because it is a folder", key);
continue;
}
@ -146,7 +146,7 @@ public class S3Utils
///////////////////////////////////////////
if(!pathMatcher.matches(Path.of(URI.create("file:///" + key))))
{
LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
// LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
continue;
}

134
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -24,20 +24,27 @@ package com.kingsrook.qqq.backend.module.filesystem;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed.StreamedETLFilesystemBackendStep;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
@ -49,10 +56,16 @@ import org.apache.commons.io.FileUtils;
*******************************************************************************/
public class TestUtils
{
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String TABLE_NAME_PERSON_LOCAL_FS = "person";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
///////////////////////////////////////////////////////////////////
// shouldn't be accessed directly, as we append a counter to it. //
@ -112,14 +125,18 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QInstance defineInstance() throws QInstanceValidationException
public static QInstance defineInstance() throws QException
{
QInstance qInstance = new QInstance();
qInstance.setAuthentication(defineAuthentication());
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addBackend(defineS3Backend());
qInstance.addTable(defineS3CSVPersonTable());
qInstance.addBackend(defineMockBackend());
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
new QInstanceValidator().validate(qInstance);
@ -159,21 +176,55 @@ public class TestUtils
public static QTableMetaData defineLocalFilesystemJSONPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_LOCAL_FS)
.withName(TABLE_NAME_PERSON_LOCAL_FS_JSON)
.withLabel("Person")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withFields(defineCommonPersonTableFields())
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("persons")
.withRecordFormat(RecordFormat.JSON)
.withCardinality(Cardinality.MANY)
.withGlob("*.json")
);
}
/*******************************************************************************
**
*******************************************************************************/
private static List<QFieldMetaData> defineCommonPersonTableFields()
{
return (List.of(
new QFieldMetaData("id", QFieldType.INTEGER),
new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"),
new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"),
new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"),
new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"),
new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"),
new QFieldMetaData("email", QFieldType.STRING)
));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemCSVPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_LOCAL_FS_CSV)
.withLabel("Person")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("id")
.withFields(defineCommonPersonTableFields())
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("persons-csv")
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
.withGlob("*.csv")
);
}
@ -202,13 +253,7 @@ public class TestUtils
.withLabel("Person S3 Table")
.withBackendName(defineS3Backend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withFields(defineCommonPersonTableFields())
.withBackendDetails(new S3TableBackendDetails()
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
@ -220,7 +265,52 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QSession getMockSession() throws QInstanceValidationException
public static QBackendMetaData defineMockBackend()
{
return (new QBackendMetaData()
.withBackendType("mock")
.withName(BACKEND_NAME_MOCK));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineMockPersonTable()
{
return (new QTableMetaData()
.withName(TABLE_NAME_PERSON_MOCK)
.withLabel("Person Mock Table")
.withBackendName(BACKEND_NAME_MOCK)
.withPrimaryKeyField("id")
.withFields(defineCommonPersonTableFields()));
}
/*******************************************************************************
**
*******************************************************************************/
private static QProcessMetaData defineStreamedLocalCsvToMockETLProcess() throws QException
{
QProcessMetaData qProcessMetaData = new StreamedETLProcess().defineProcessMetaData();
qProcessMetaData.setName(PROCESS_NAME_STREAMED_ETL);
QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
backendStep.setCode(new QCodeReference(StreamedETLFilesystemBackendStep.class));
backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_SOURCE_TABLE).setDefaultValue(TABLE_NAME_PERSON_LOCAL_FS_CSV);
backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_DESTINATION_TABLE).setDefaultValue(TABLE_NAME_PERSON_MOCK);
return (qProcessMetaData);
}
/*******************************************************************************
**
*******************************************************************************/
public static QSession getMockSession() throws QException
{
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
return (mockAuthenticationModule.createSession(defineInstance(), null));

8
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java
View File

@ -70,7 +70,7 @@ public class FilesystemBackendModuleTest
public void testDeleteFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
@ -94,7 +94,7 @@ public class FilesystemBackendModuleTest
public void testDeleteFileDoesNotExist() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
@ -120,7 +120,7 @@ public class FilesystemBackendModuleTest
public void testMoveFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
@ -157,7 +157,7 @@ public class FilesystemBackendModuleTest
public void testMoveFileDoesNotExit() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));

37
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java
View File

@ -80,7 +80,8 @@ public class FilesystemActionTest
fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module");
}
writePersonFiles(baseDirectory);
writePersonJSONFiles(baseDirectory);
writePersonCSVFiles(baseDirectory);
}
@ -88,7 +89,7 @@ public class FilesystemActionTest
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writePersonFiles(File baseDirectory) throws IOException
private void writePersonJSONFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
@ -118,6 +119,38 @@ public class FilesystemActionTest
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writePersonCSVFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
{
if (StringUtils.hasContent(details.getBasePath()))
{
fullPath += File.separatorChar + details.getBasePath();
}
}
fullPath += File.separatorChar;
String csvData1 = """
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
"1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
"2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
""";
FileUtils.writeStringToFile(new File(fullPath + "FILE-1.csv"), csvData1);
String csvData2 = """
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
"3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
"4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
"5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com\""""; // intentionally no \n at EOL here
FileUtils.writeStringToFile(new File(fullPath + "FILE-2.csv"), csvData2);
}
/*******************************************************************************
**
*******************************************************************************/

2
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java
View File

@ -71,7 +71,7 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
QueryInput queryInput = new QueryInput();
QInstance instance = TestUtils.defineInstance();
QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference()
.withName(ValueUpshifter.class.getName())
.withCodeType(QCodeType.JAVA)

8
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java
View File

@ -24,7 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -44,7 +44,7 @@ class FilesystemBackendMetaDataTest
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
public void testSerializingToJson() throws QException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
@ -62,7 +62,7 @@ class FilesystemBackendMetaDataTest
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
public void testDeserializingFromJson() throws IOException, QException
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
@ -71,6 +71,8 @@ class FilesystemBackendMetaDataTest
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized.getBackends()).usingRecursiveComparison()
// TODO seeing occassional flaps on this field - where it can be null 1 out of 10 runs... unclear why.
.ignoringFields("mock.backendType")
.isEqualTo(qInstance.getBackends());
}
}

4
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java
View File

@ -198,7 +198,7 @@ public class BasicETLCleanupSourceFilesStepTest
runBackendStepInput.setProcessName(qProcessMetaData.getName());
// runFunctionRequest.setRecords(records);
runBackendStepInput.setSession(TestUtils.getMockSession());
runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
runBackendStepInput.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runBackendStepInput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
@ -219,7 +219,7 @@ public class BasicETLCleanupSourceFilesStepTest
private String getRandomFilePathPersonTable(QInstance qInstance)
{
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails();
FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON).getBackendDetails();
String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath();
String filePath = tablePath + File.separator + UUID.randomUUID();
return filePath;

62
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStepTest.java Normal file
View File

@ -0,0 +1,62 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic.BasicETLCollectSourceFileNamesStep;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Unit test for StreamedETLFilesystemBackendStep
*******************************************************************************/
class StreamedETLFilesystemBackendStepTest extends FilesystemActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void runFullProcess() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(TestUtils.getMockSession());
runProcessInput.setProcessName(TestUtils.PROCESS_NAME_STREAMED_ETL);
RunProcessOutput output = new RunProcessAction().execute(runProcessInput);
String sourceFilePaths = ValueUtils.getValueAsString(output.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS));
assertThat(sourceFilePaths)
.contains("FILE-1.csv")
.contains("FILE-2.csv");
}
}

3
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java
View File

@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -60,7 +59,7 @@ public class S3QueryActionTest extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
private QueryInput initQueryRequest() throws QInstanceValidationException
private QueryInput initQueryRequest() throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setInstance(TestUtils.defineInstance());

9
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java
View File

@ -22,9 +22,8 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -44,7 +43,7 @@ class S3BackendMetaDataTest
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
public void testSerializingToJson() throws QException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
@ -62,7 +61,7 @@ class S3BackendMetaDataTest
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
public void testDeserializingFromJson() throws Exception
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
@ -71,6 +70,8 @@ class S3BackendMetaDataTest
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized.getBackends()).usingRecursiveComparison()
// TODO seeing occassional flaps on this field - where it can be null 1 out of 10 runs... unclear why.
.ignoringFields("mock.backendType")
.isEqualTo(qInstance.getBackends());
}
}

16
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
View File

@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
@ -94,7 +95,8 @@ public abstract class AbstractRDBMSAction
/*******************************************************************************
** Handle obvious problems with values - like empty string for integer should be null.
** Handle obvious problems with values - like empty string for integer should be null,
** and type conversions that we can do "better" than jdbc...
**
*******************************************************************************/
protected Serializable scrubValue(QFieldMetaData field, Serializable value, boolean isInsert)
@ -108,6 +110,18 @@ public abstract class AbstractRDBMSAction
}
}
//////////////////////////////////////////////////////////////////////////////
// value utils is good at making values from strings - jdbc, not as much... //
//////////////////////////////////////////////////////////////////////////////
if(field.getType().equals(QFieldType.DATE) && value instanceof String)
{
value = ValueUtils.getValueAsLocalDate(value);
}
else if(field.getType().equals(QFieldType.DECIMAL) && value instanceof String)
{
value = ValueUtils.getValueAsBigDecimal(value);
}
return (value);
}

45
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
View File

@ -28,6 +28,7 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@ -92,7 +93,20 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
List<QRecord> outputRecords = new ArrayList<>();
rs.setRecords(outputRecords);
try(Connection connection = getConnection(insertInput))
Connection connection;
boolean needToCloseConnection = false;
if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
{
LOG.debug("Using connection from insertInput [" + rdbmsTransaction.getConnection() + "]");
connection = rdbmsTransaction.getConnection();
}
else
{
connection = getConnection(insertInput);
needToCloseConnection = true;
}
try
{
for(List<QRecord> page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE))
{
@ -130,6 +144,13 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
}
}
}
finally
{
if(needToCloseConnection)
{
connection.close();
}
}
return rs;
}
@ -139,4 +160,26 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QBackendTransaction openTransaction(InsertInput insertInput) throws QException
{
try
{
LOG.info("Opening transaction");
Connection connection = getConnection(insertInput);
return (new RDBMSTransaction(connection));
}
catch(Exception e)
{
throw new QException("Error opening transaction: " + e.getMessage(), e);
}
}
}

5
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
View File

@ -189,8 +189,13 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
}
case DATE:
{
// todo - queryManager.getLocalDate?
return (QueryManager.getDate(resultSet, i));
}
case TIME:
{
return (QueryManager.getLocalTime(resultSet, i));
}
case DATE_TIME:
{
return (QueryManager.getLocalDateTime(resultSet, i));

130
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java Normal file
View File

@ -0,0 +1,130 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.rdbms.actions;
import java.sql.Connection;
import java.sql.SQLException;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** RDBMS implementation of backend transaction.
**
** Stores a jdbc connection, which is set to autoCommit(false).
*******************************************************************************/
public class RDBMSTransaction extends QBackendTransaction
{
private static final Logger LOG = LogManager.getLogger(RDBMSTransaction.class);
private Connection connection;
/*******************************************************************************
**
*******************************************************************************/
public RDBMSTransaction(Connection connection) throws SQLException
{
connection.setAutoCommit(false);
this.connection = connection;
}
/*******************************************************************************
** Getter for connection
**
*******************************************************************************/
public Connection getConnection()
{
return connection;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void commit() throws QException
{
try
{
RDBMSTransaction.LOG.info("Committing transaction");
connection.commit();
RDBMSTransaction.LOG.info("Commit complete");
}
catch(Exception e)
{
RDBMSTransaction.LOG.error("Error committing transaction", e);
throw new QException("Error committing transaction: " + e.getMessage(), e);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void rollback() throws QException
{
try
{
RDBMSTransaction.LOG.info("Rolling back transaction");
connection.rollback();
RDBMSTransaction.LOG.info("Rollback complete");
}
catch(Exception e)
{
RDBMSTransaction.LOG.error("Error rolling back transaction", e);
throw new QException("Error rolling back transaction: " + e.getMessage(), e);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void close()
{
try
{
if(connection.isClosed())
{
return;
}
connection.close();
}
catch(Exception e)
{
LOG.error("Error closing connection - possible jdbc connection leak", e);
}
}
}

138
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
@ -35,6 +36,7 @@ import java.sql.Types;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
@ -47,6 +49,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.commons.lang.NotImplementedException;
@ -121,7 +124,10 @@ public class QueryManager
statement.execute();
resultSet = statement.getResultSet();
processor.processResultSet(resultSet);
if(processor != null)
{
processor.processResultSet(resultSet);
}
}
finally
{
@ -183,8 +189,6 @@ public class QueryManager
@SuppressWarnings("unchecked")
public static <T> T executeStatementForSingleValue(Connection connection, Class<T> returnClass, String sql, Object... params) throws SQLException
{
throw (new NotImplementedException());
/*
PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params);
statement.execute();
ResultSet resultSet = statement.getResultSet();
@ -233,7 +237,6 @@ public class QueryManager
{
return (null);
}
*/
}
@ -685,6 +688,11 @@ public class QueryManager
bindParam(statement, index, l.intValue());
return (1);
}
else if(value instanceof Double d)
{
bindParam(statement, index, d.doubleValue());
return (1);
}
else if(value instanceof String s)
{
bindParam(statement, index, s);
@ -743,10 +751,14 @@ public class QueryManager
}
else if(value instanceof LocalDate ld)
{
ZoneOffset offset = OffsetDateTime.now().getOffset();
long epochMillis = ld.atStartOfDay().toEpochSecond(offset) * MS_PER_SEC;
Timestamp timestamp = new Timestamp(epochMillis);
statement.setTimestamp(index, timestamp);
java.sql.Date date = new java.sql.Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth());
statement.setDate(index, date);
return (1);
}
else if(value instanceof LocalTime lt)
{
java.sql.Time time = new java.sql.Time(lt.getHour(), lt.getMinute(), lt.getSecond());
statement.setTime(index, time);
return (1);
}
else if(value instanceof OffsetDateTime odt)
@ -851,6 +863,23 @@ public class QueryManager
/*******************************************************************************
*
*******************************************************************************/
public static void bindParam(PreparedStatement statement, int index, Double value) throws SQLException
{
if(value == null)
{
statement.setNull(index, Types.DOUBLE);
}
else
{
statement.setDouble(index, value);
}
}
/*******************************************************************************
*
*******************************************************************************/
@ -1199,6 +1228,67 @@ public class QueryManager
/*******************************************************************************
**
*******************************************************************************/
public static LocalTime getLocalTime(ResultSet resultSet, int column) throws SQLException
{
String timeString = resultSet.getString(column);
if(resultSet.wasNull())
{
return (null);
}
return stringToLocalTime(timeString);
}
/*******************************************************************************
**
*******************************************************************************/
public static LocalTime getLocalTime(ResultSet resultSet, String column) throws SQLException
{
String timeString = resultSet.getString(column);
if(resultSet.wasNull())
{
return (null);
}
return stringToLocalTime(timeString);
}
/*******************************************************************************
**
*******************************************************************************/
private static LocalTime stringToLocalTime(String timeString) throws SQLException
{
if(!StringUtils.hasContent(timeString))
{
return (null);
}
String[] parts = timeString.split(":");
if(parts.length == 1)
{
return LocalTime.of(Integer.parseInt(parts[0]), 0);
}
if(parts.length == 2)
{
return LocalTime.of(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]));
}
else if(parts.length == 3)
{
return LocalTime.of(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2]));
}
else
{
throw (new SQLException("Unable to parse time value [" + timeString + "] to LocalTime"));
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -1305,6 +1395,38 @@ public class QueryManager
/*******************************************************************************
**
*******************************************************************************/
public static Instant getInstant(ResultSet resultSet, String column) throws SQLException
{
Timestamp value = resultSet.getTimestamp(column);
if(resultSet.wasNull())
{
return (null);
}
return (value.toInstant());
}
/*******************************************************************************
**
*******************************************************************************/
public static Instant getInstant(ResultSet resultSet, int column) throws SQLException
{
Timestamp value = resultSet.getTimestamp(column);
if(resultSet.wasNull())
{
return (null);
}
return (value.toInstant());
}
/*******************************************************************************
**
*******************************************************************************/

47
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java
View File

@ -22,12 +22,22 @@
package com.kingsrook.qqq.backend.module.rdbms;
import java.io.InputStream;
import java.sql.Connection;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
import org.apache.commons.io.IOUtils;
import static junit.framework.Assert.assertNotNull;
/*******************************************************************************
@ -40,6 +50,29 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public static void primeTestDatabase(String sqlFileName) throws Exception
{
ConnectionManager connectionManager = new ConnectionManager();
try(Connection connection = connectionManager.getConnection(TestUtils.defineBackend()))
{
InputStream primeTestDatabaseSqlStream = RDBMSActionTest.class.getResourceAsStream("/" + sqlFileName);
assertNotNull(primeTestDatabaseSqlStream);
List<String> lines = (List<String>) IOUtils.readLines(primeTestDatabaseSqlStream);
lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList();
String joinedSQL = String.join("\n", lines);
for(String sql : joinedSQL.split(";"))
{
QueryManager.executeUpdate(connection, sql);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -48,11 +81,25 @@ public class TestUtils
QInstance qInstance = new QInstance();
qInstance.addBackend(defineBackend());
qInstance.addTable(defineTablePerson());
qInstance.setAuthentication(defineAuthentication());
return (qInstance);
}
/*******************************************************************************
** Define the authentication used in standard tests - using 'mock' type.
**
*******************************************************************************/
public static QAuthenticationMetaData defineAuthentication()
{
return new QAuthenticationMetaData()
.withName("mock")
.withType(QAuthenticationType.MOCK);
}
/*******************************************************************************
**
*******************************************************************************/

28
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java
View File

@ -22,15 +22,11 @@
package com.kingsrook.qqq.backend.module.rdbms.actions;
import java.io.InputStream;
import java.sql.Connection;
import java.util.List;
import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.AfterEach;
import static junit.framework.Assert.assertNotNull;
/*******************************************************************************
@ -57,33 +53,11 @@ public class RDBMSActionTest
*******************************************************************************/
protected void primeTestDatabase() throws Exception
{
primeTestDatabase("prime-test-database.sql");
TestUtils.primeTestDatabase("prime-test-database.sql");
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("unchecked")
protected void primeTestDatabase(String sqlFileName) throws Exception
{
ConnectionManager connectionManager = new ConnectionManager();
try(Connection connection = connectionManager.getConnection(TestUtils.defineBackend()))
{
InputStream primeTestDatabaseSqlStream = RDBMSActionTest.class.getResourceAsStream("/" + sqlFileName);
assertNotNull(primeTestDatabaseSqlStream);
List<String> lines = (List<String>) IOUtils.readLines(primeTestDatabaseSqlStream);
lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList();
String joinedSQL = String.join("\n", lines);
for(String sql : joinedSQL.split(";"))
{
QueryManager.executeUpdate(connection, sql);
}
}
}
/*******************************************************************************
**

2
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java
View File

@ -151,7 +151,7 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest
//////////////////////////////////////////////////////////////////
// load the parent-child tables, with foreign keys and instance //
//////////////////////////////////////////////////////////////////
super.primeTestDatabase("prime-test-database-parent-child-tables.sql");
TestUtils.primeTestDatabase("prime-test-database-parent-child-tables.sql");
DeleteInput deleteInput = initChildTableInstanceAndDeleteRequest();
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////

26
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java
View File

@ -23,16 +23,20 @@ package com.kingsrook.qqq.backend.module.rdbms.actions;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
@ -416,7 +420,29 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
QueryInput queryInput = new QueryInput();
queryInput.setInstance(TestUtils.defineInstance());
queryInput.setTableName(TestUtils.defineTablePerson().getName());
queryInput.setSession(new QSession());
return queryInput;
}
/*******************************************************************************
** This doesn't really test any RDBMS code, but is a checkpoint that the core
** module is populating displayValues when it performs the system-level query action.
*******************************************************************************/
@Test
public void testThatDisplayValuesGetSetGoingThroughQueryAction() throws QException
{
QueryInput queryInput = initQueryRequest();
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(5, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
for(QRecord record : queryOutput.getRecords())
{
assertThat(record.getValues()).isNotEmpty();
assertThat(record.getDisplayValues()).isNotEmpty();
assertThat(record.getErrors()).isEmpty();
}
}
}

96
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransactionTest.java Normal file
View File

@ -0,0 +1,96 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.rdbms.actions;
import java.sql.Connection;
import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for RDBMSTransaction
*******************************************************************************/
class RDBMSTransactionTest
{
private final String testToken = getClass().getName();
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
protected void beforeEach() throws Exception
{
TestUtils.primeTestDatabase("prime-test-database.sql");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCommit() throws Exception
{
ConnectionManager connectionManager = new ConnectionManager();
Connection connection = connectionManager.getConnection(TestUtils.defineBackend());
Integer preCount = QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM person");
Connection connectionForTransaction = connectionManager.getConnection(TestUtils.defineBackend());
RDBMSTransaction transaction = new RDBMSTransaction(connectionForTransaction);
QueryManager.executeUpdate(transaction.getConnection(), "INSERT INTO person (first_name, last_name, email) VALUES (?, ?, ?)", testToken, testToken, testToken);
transaction.commit();
Integer postCount = QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM person");
assertEquals(preCount + 1, postCount);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testRollback() throws Exception
{
ConnectionManager connectionManager = new ConnectionManager();
Connection connection = connectionManager.getConnection(TestUtils.defineBackend());
Integer preCount = QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM person");
Connection connectionForTransaction = connectionManager.getConnection(TestUtils.defineBackend());
RDBMSTransaction transaction = new RDBMSTransaction(connectionForTransaction);
QueryManager.executeUpdate(transaction.getConnection(), "INSERT INTO person (first_name, last_name, email) VALUES (?, ?, ?)", testToken, testToken, testToken);
transaction.rollback();
Integer postCount = QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM person");
assertEquals(preCount, postCount);
}
}

162
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java
View File

@ -32,6 +32,7 @@ import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.Month;
import java.time.OffsetDateTime;
import java.util.GregorianCalendar;
@ -59,7 +60,16 @@ class QueryManagerTest
void beforeEach() throws SQLException
{
Connection connection = getConnection();
QueryManager.executeUpdate(connection, "CREATE TABLE t (i INTEGER, dt DATETIME, c CHAR(1), d DATE)");
QueryManager.executeUpdate(connection, """
CREATE TABLE test_table
(
int_col INTEGER,
datetime_col DATETIME,
char_col CHAR(1),
date_col DATE,
time_col TIME
)
""");
}
@ -71,7 +81,7 @@ class QueryManagerTest
void afterEach() throws SQLException
{
Connection connection = getConnection();
QueryManager.executeUpdate(connection, "DROP TABLE t");
QueryManager.executeUpdate(connection, "DROP TABLE test_table");
}
@ -95,7 +105,7 @@ class QueryManagerTest
{
long ctMillis = System.currentTimeMillis();
Connection connection = getConnection();
PreparedStatement ps = connection.prepareStatement("UPDATE t SET i = ? WHERE i > 0");
PreparedStatement ps = connection.prepareStatement("UPDATE test_table SET int_col = ? WHERE int_col > 0");
///////////////////////////////////////////////////////////////////////////////
// these calls - we just want to assert that they don't throw any exceptions //
@ -149,37 +159,37 @@ class QueryManagerTest
void testGetValueMethods() throws SQLException
{
Connection connection = getConnection();
QueryManager.executeUpdate(connection, "INSERT INTO t (i, dt, c) VALUES (1, now(), 'A')");
PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from t");
QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, datetime_col, char_col) VALUES (1, now(), 'A')");
PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from test_table");
preparedStatement.execute();
ResultSet rs = preparedStatement.getResultSet();
rs.next();
assertEquals(1, QueryManager.getInteger(rs, "i"));
assertEquals(1, QueryManager.getInteger(rs, "int_col"));
assertEquals(1, QueryManager.getInteger(rs, 1));
assertEquals(1L, QueryManager.getLong(rs, "i"));
assertEquals(1L, QueryManager.getLong(rs, "int_col"));
assertEquals(1L, QueryManager.getLong(rs, 1));
assertArrayEquals(new byte[] { 0, 0, 0, 1 }, QueryManager.getByteArray(rs, "i"));
assertArrayEquals(new byte[] { 0, 0, 0, 1 }, QueryManager.getByteArray(rs, "int_col"));
assertArrayEquals(new byte[] { 0, 0, 0, 1 }, QueryManager.getByteArray(rs, 1));
assertEquals(1, QueryManager.getObject(rs, "i"));
assertEquals(1, QueryManager.getObject(rs, "int_col"));
assertEquals(1, QueryManager.getObject(rs, 1));
assertEquals(BigDecimal.ONE, QueryManager.getBigDecimal(rs, "i"));
assertEquals(BigDecimal.ONE, QueryManager.getBigDecimal(rs, "int_col"));
assertEquals(BigDecimal.ONE, QueryManager.getBigDecimal(rs, 1));
assertEquals(true, QueryManager.getBoolean(rs, "i"));
assertEquals(true, QueryManager.getBoolean(rs, "int_col"));
assertEquals(true, QueryManager.getBoolean(rs, 1));
assertNotNull(QueryManager.getDate(rs, "dt"));
assertNotNull(QueryManager.getDate(rs, "datetime_col"));
assertNotNull(QueryManager.getDate(rs, 2));
assertNotNull(QueryManager.getCalendar(rs, "dt"));
assertNotNull(QueryManager.getCalendar(rs, "datetime_col"));
assertNotNull(QueryManager.getCalendar(rs, 2));
assertNotNull(QueryManager.getLocalDate(rs, "dt"));
assertNotNull(QueryManager.getLocalDate(rs, "datetime_col"));
assertNotNull(QueryManager.getLocalDate(rs, 2));
assertNotNull(QueryManager.getLocalDateTime(rs, "dt"));
assertNotNull(QueryManager.getLocalDateTime(rs, "datetime_col"));
assertNotNull(QueryManager.getLocalDateTime(rs, 2));
assertNotNull(QueryManager.getOffsetDateTime(rs, "dt"));
assertNotNull(QueryManager.getOffsetDateTime(rs, "datetime_col"));
assertNotNull(QueryManager.getOffsetDateTime(rs, 2));
assertNotNull(QueryManager.getTimestamp(rs, "dt"));
assertNotNull(QueryManager.getTimestamp(rs, "datetime_col"));
assertNotNull(QueryManager.getTimestamp(rs, 2));
assertEquals("A", QueryManager.getObject(rs, "c"));
assertEquals("A", QueryManager.getObject(rs, "char_col"));
assertEquals("A", QueryManager.getObject(rs, 3));
}
@ -192,37 +202,37 @@ class QueryManagerTest
void testGetValueMethodsReturningNull() throws SQLException
{
Connection connection = getConnection();
QueryManager.executeUpdate(connection, "INSERT INTO t (i, dt, c) VALUES (null, null, null)");
PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from t");
QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, datetime_col, char_col) VALUES (null, null, null)");
PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from test_table");
preparedStatement.execute();
ResultSet rs = preparedStatement.getResultSet();
rs.next();
assertNull(QueryManager.getInteger(rs, "i"));
assertNull(QueryManager.getInteger(rs, "int_col"));
assertNull(QueryManager.getInteger(rs, 1));
assertNull(QueryManager.getLong(rs, "i"));
assertNull(QueryManager.getLong(rs, "int_col"));
assertNull(QueryManager.getLong(rs, 1));
assertNull(QueryManager.getByteArray(rs, "i"));
assertNull(QueryManager.getByteArray(rs, "int_col"));
assertNull(QueryManager.getByteArray(rs, 1));
assertNull(QueryManager.getObject(rs, "i"));
assertNull(QueryManager.getObject(rs, "int_col"));
assertNull(QueryManager.getObject(rs, 1));
assertNull(QueryManager.getBigDecimal(rs, "i"));
assertNull(QueryManager.getBigDecimal(rs, "int_col"));
assertNull(QueryManager.getBigDecimal(rs, 1));
assertNull(QueryManager.getBoolean(rs, "i"));
assertNull(QueryManager.getBoolean(rs, "int_col"));
assertNull(QueryManager.getBoolean(rs, 1));
assertNull(QueryManager.getDate(rs, "dt"));
assertNull(QueryManager.getDate(rs, "datetime_col"));
assertNull(QueryManager.getDate(rs, 2));
assertNull(QueryManager.getCalendar(rs, "dt"));
assertNull(QueryManager.getCalendar(rs, "datetime_col"));
assertNull(QueryManager.getCalendar(rs, 2));
assertNull(QueryManager.getLocalDate(rs, "dt"));
assertNull(QueryManager.getLocalDate(rs, "datetime_col"));
assertNull(QueryManager.getLocalDate(rs, 2));
assertNull(QueryManager.getLocalDateTime(rs, "dt"));
assertNull(QueryManager.getLocalDateTime(rs, "datetime_col"));
assertNull(QueryManager.getLocalDateTime(rs, 2));
assertNull(QueryManager.getOffsetDateTime(rs, "dt"));
assertNull(QueryManager.getOffsetDateTime(rs, "datetime_col"));
assertNull(QueryManager.getOffsetDateTime(rs, 2));
assertNull(QueryManager.getTimestamp(rs, "dt"));
assertNull(QueryManager.getTimestamp(rs, "datetime_col"));
assertNull(QueryManager.getTimestamp(rs, 2));
assertNull(QueryManager.getObject(rs, "c"));
assertNull(QueryManager.getObject(rs, "char_col"));
assertNull(QueryManager.getObject(rs, 3));
}
@ -236,9 +246,9 @@ class QueryManagerTest
void testLocalDate() throws SQLException
{
Connection connection = getConnection();
QueryManager.executeUpdate(connection, "INSERT INTO t (d) VALUES (?)", LocalDate.of(2013, Month.OCTOBER, 1));
QueryManager.executeUpdate(connection, "INSERT INTO test_table (date_col) VALUES (?)", LocalDate.of(2013, Month.OCTOBER, 1));
PreparedStatement preparedStatement = connection.prepareStatement("SELECT d from t");
PreparedStatement preparedStatement = connection.prepareStatement("SELECT date_col from test_table");
preparedStatement.execute();
ResultSet rs = preparedStatement.getResultSet();
rs.next();
@ -268,4 +278,86 @@ class QueryManagerTest
assertEquals(0, offsetDateTime.getMinute(), "Minute value");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testLocalTime() throws SQLException
{
Connection connection = getConnection();
////////////////////////////////////
// insert one just hour & minutes //
////////////////////////////////////
QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, time_col) VALUES (?, ?)", 1, LocalTime.of(10, 42));
PreparedStatement preparedStatement = connection.prepareStatement("SELECT time_col from test_table where int_col=1");
preparedStatement.execute();
ResultSet rs = preparedStatement.getResultSet();
rs.next();
LocalTime localTime = QueryManager.getLocalTime(rs, 1);
assertEquals(10, localTime.getHour(), "Hour value");
assertEquals(42, localTime.getMinute(), "Minute value");
assertEquals(0, localTime.getSecond(), "Second value");
localTime = QueryManager.getLocalTime(rs, "time_col");
assertEquals(10, localTime.getHour(), "Hour value");
assertEquals(42, localTime.getMinute(), "Minute value");
assertEquals(0, localTime.getSecond(), "Second value");
/////////////////////////////////
// now insert one with seconds //
/////////////////////////////////
QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, time_col) VALUES (?, ?)", 2, LocalTime.of(10, 42, 59));
preparedStatement = connection.prepareStatement("SELECT time_col from test_table where int_col=2");
preparedStatement.execute();
rs = preparedStatement.getResultSet();
rs.next();
localTime = QueryManager.getLocalTime(rs, 1);
assertEquals(10, localTime.getHour(), "Hour value");
assertEquals(42, localTime.getMinute(), "Minute value");
assertEquals(59, localTime.getSecond(), "Second value");
localTime = QueryManager.getLocalTime(rs, "time_col");
assertEquals(10, localTime.getHour(), "Hour value");
assertEquals(42, localTime.getMinute(), "Minute value");
assertEquals(59, localTime.getSecond(), "Second value");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testExecuteStatementForSingleValue() throws SQLException
{
Connection connection = getConnection();
QueryManager.executeUpdate(connection, """
INSERT INTO test_table
( int_col, datetime_col, char_col, date_col, time_col )
VALUES
( 47, '2022-08-10 19:22:08', 'Q', '2022-08-10', '19:22:08')
""");
assertEquals(null, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT int_col FROM test_table WHERE int_col = -1"));
assertEquals(1, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM test_table"));
assertEquals(47, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT int_col FROM test_table"));
assertEquals("Q", QueryManager.executeStatementForSingleValue(connection, String.class, "SELECT char_col FROM test_table"));
assertEquals(new BigDecimal("1.1"), QueryManager.executeStatementForSingleValue(connection, BigDecimal.class, "SELECT 1.1 FROM test_table"));
assertEquals(1, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT 1.1 FROM test_table"));
QueryManager.executeUpdate(connection, """
INSERT INTO test_table
( int_col, datetime_col, char_col, date_col, time_col )
VALUES
( null, null, null, null, null)
""");
assertEquals(null, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT int_col FROM test_table WHERE int_col IS NULL"));
}
}

39
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -46,10 +46,12 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
@ -121,7 +123,7 @@ public class QJavalinImplementation
/*******************************************************************************
**
*******************************************************************************/
public static void main(String[] args)
public static void main(String[] args) throws QInstanceValidationException
{
QInstance qInstance = new QInstance();
// todo - parse args to look up metaData and prime instance
@ -135,9 +137,10 @@ public class QJavalinImplementation
/*******************************************************************************
**
*******************************************************************************/
public QJavalinImplementation(QInstance qInstance)
public QJavalinImplementation(QInstance qInstance) throws QInstanceValidationException
{
QJavalinImplementation.qInstance = qInstance;
new QInstanceValidator().validate(qInstance);
}
@ -251,11 +254,39 @@ public class QJavalinImplementation
try
{
Map<String, String> authenticationContext = new HashMap<>();
authenticationContext.put(SESSION_ID_COOKIE_NAME, context.cookie(SESSION_ID_COOKIE_NAME));
/////////////////////////////////////////////////////////////////////////////////
// look for a token in either the sessionId cookie, or an Authorization header //
/////////////////////////////////////////////////////////////////////////////////
String sessionIdCookieValue = context.cookie(SESSION_ID_COOKIE_NAME);
if(StringUtils.hasContent(sessionIdCookieValue))
{
authenticationContext.put(SESSION_ID_COOKIE_NAME, sessionIdCookieValue);
}
else
{
String authorizationHeaderValue = context.header("Authorization");
if (authorizationHeaderValue != null)
{
String bearerPrefix = "Bearer ";
if(authorizationHeaderValue.startsWith(bearerPrefix))
{
authorizationHeaderValue = authorizationHeaderValue.replaceFirst(bearerPrefix, "");
}
authenticationContext.put(SESSION_ID_COOKIE_NAME, authorizationHeaderValue);
}
}
QSession session = authenticationModule.createSession(qInstance, authenticationContext);
input.setSession(session);
context.cookie(SESSION_ID_COOKIE_NAME, session.getIdReference(), SESSION_COOKIE_AGE);
/////////////////////////////////////////////////////////////////////////////////
// if we got a session id cookie in, then send it back with updated cookie age //
/////////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(sessionIdCookieValue))
{
context.cookie(SESSION_ID_COOKIE_NAME, session.getIdReference(), SESSION_COOKIE_AGE);
}
}
catch(QAuthenticationException qae)
{

3
qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinTestBase.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.javalin;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@ -44,7 +45,7 @@ public class QJavalinTestBase
**
*******************************************************************************/
@BeforeAll
public static void beforeAll()
public static void beforeAll() throws QInstanceValidationException
{
qJavalinImplementation = new QJavalinImplementation(TestUtils.defineInstance());
QJavalinProcessHandler.setAsyncStepTimeoutMillis(250);

4
qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.frontend.picocli;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@ -407,7 +408,8 @@ public class QCommandBuilder
case INTEGER -> Integer.class;
case DECIMAL -> BigDecimal.class;
case DATE -> LocalDate.class;
// case TIME -> LocalTime.class;
case TIME -> LocalTime.class;
case BOOLEAN -> Boolean.class;
case DATE_TIME -> LocalDateTime.class;
case BLOB -> byte[].class;
};

1
qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.sampleapp;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.javalin.QJavalinImplementation;
import io.javalin.Javalin;

109
qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java
View File

@ -26,23 +26,28 @@ import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
@ -59,7 +64,7 @@ import io.github.cdimascio.dotenv.Dotenv;
*******************************************************************************/
public class SampleMetaDataProvider
{
public static boolean USE_MYSQL = false;
public static boolean USE_MYSQL = true;
public static final String RDBMS_BACKEND_NAME = "rdbms";
public static final String FILESYSTEM_BACKEND_NAME = "filesystem";
@ -68,12 +73,20 @@ public class SampleMetaDataProvider
// public static final String AUTH0_BASE_URL = "https://kingsrook.us.auth0.com/";
public static final String AUTH0_BASE_URL = "https://nutrifresh-one-development.us.auth0.com/";
public static final String APP_NAME_GREETINGS = "greetingsApp";
public static final String APP_NAME_PEOPLE = "peopleApp";
public static final String APP_NAME_MISCELLANEOUS = "miscellaneous";
public static final String PROCESS_NAME_GREET = "greet";
public static final String PROCESS_NAME_GREET_INTERACTIVE = "greetInteractive";
public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep";
public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow";
public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive";
public static final String TABLE_NAME_PERSON = "person";
public static final String TABLE_NAME_CARRIER = "carrier";
public static final String TABLE_NAME_CITY = "city";
public static final String STEP_NAME_SLEEPER = "sleeper";
public static final String STEP_NAME_THROWER = "thrower";
@ -101,11 +114,46 @@ public class SampleMetaDataProvider
qInstance.addProcess(defineProcessScreenThenSleep());
qInstance.addProcess(defineProcessSimpleThrow());
defineApps(qInstance);
return (qInstance);
}
/*******************************************************************************
**
*******************************************************************************/
private static void defineApps(QInstance qInstance)
{
qInstance.addApp(new QAppMetaData()
.withName(APP_NAME_GREETINGS)
.withIcon(new QIcon().withName("emoji_people"))
.withChild(qInstance.getProcess(PROCESS_NAME_GREET)
.withIcon(new QIcon().withName("emoji_people")))
.withChild(qInstance.getTable(TABLE_NAME_PERSON)
.withIcon(new QIcon().withName("person")))
.withChild(qInstance.getTable(TABLE_NAME_CITY)
.withIcon(new QIcon().withName("location_city")))
.withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE))
.withIcon(new QIcon().withName("waving_hand")));
qInstance.addApp(new QAppMetaData()
.withName(APP_NAME_PEOPLE)
.withIcon(new QIcon().withName("person"))
.withChild(qInstance.getApp(APP_NAME_GREETINGS)));
qInstance.addApp(new QAppMetaData()
.withName(APP_NAME_MISCELLANEOUS)
.withIcon(new QIcon().withName("stars"))
.withChild(qInstance.getTable(TABLE_NAME_CARRIER).withIcon(new QIcon("local_shipping")))
.withChild(qInstance.getProcess(PROCESS_NAME_SIMPLE_SLEEP))
.withChild(qInstance.getProcess(PROCESS_NAME_SLEEP_INTERACTIVE))
.withChild(qInstance.getProcess(PROCESS_NAME_SIMPLE_THROW)));
}
/*******************************************************************************
**
*******************************************************************************/
@ -166,22 +214,28 @@ public class SampleMetaDataProvider
public static QTableMetaData defineTableCarrier()
{
QTableMetaData table = new QTableMetaData();
table.setName("carrier");
table.setName(TABLE_NAME_CARRIER);
table.setBackendName(RDBMS_BACKEND_NAME);
table.setPrimaryKeyField("id");
table.setRecordLabelFormat("%s");
table.setRecordLabelFields(List.of("name"));
table.addField(new QFieldMetaData("id", QFieldType.INTEGER));
table.addField(new QFieldMetaData("name", QFieldType.STRING)
.withIsRequired(true));
table.addField(new QFieldMetaData("company_code", QFieldType.STRING) // todo enum
table.addField(new QFieldMetaData("company_code", QFieldType.STRING) // todo PVS
.withLabel("Company")
.withIsRequired(true)
.withBackendName("comp_code"));
.withBackendName("company_code"));
table.addField(new QFieldMetaData("service_level", QFieldType.STRING)
.withIsRequired(true)); // todo enum
table.addField(new QFieldMetaData("service_level", QFieldType.STRING) // todo PVS
.withLabel("Service Level")
.withIsRequired(true));
table.addSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "name")));
table.addSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("company_code", "service_level")));
return (table);
}
@ -193,18 +247,31 @@ public class SampleMetaDataProvider
*******************************************************************************/
public static QTableMetaData defineTablePerson()
{
return new QTableMetaData()
.withName("person")
QTableMetaData qTableMetaData = new QTableMetaData()
.withName(TABLE_NAME_PERSON)
.withLabel("Person")
.withBackendName(RDBMS_BACKEND_NAME)
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withRecordLabelFormat("%s %s")
.withRecordLabelFields(List.of("firstName", "lastName"))
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date").withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date").withIsEditable(false))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name").withIsRequired(true))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name").withIsRequired(true))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING));
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary").withDisplayFormat(DisplayFormat.CURRENCY))
.withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked").withDisplayFormat(DisplayFormat.COMMAS))
.withSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "firstName", "lastName")))
.withSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("email", "birthDate")))
.withSection(new QFieldSection("employmentInfo", "Employment Info", new QIcon("work"), Tier.T2, List.of("annualSalary", "daysWorked")))
.withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData);
return (qTableMetaData);
}
@ -215,7 +282,7 @@ public class SampleMetaDataProvider
public static QTableMetaData defineTableCityFile()
{
return new QTableMetaData()
.withName("city")
.withName(TABLE_NAME_CITY)
.withLabel("Cities")
.withIsHidden(true)
.withBackendName(FILESYSTEM_BACKEND_NAME)
@ -240,7 +307,7 @@ public class SampleMetaDataProvider
return new QProcessMetaData()
.withName(PROCESS_NAME_GREET)
.withLabel("Greet People")
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.withIsHidden(true)
.addStep(new QBackendStepMetaData()
.withName("prepare")
@ -249,14 +316,14 @@ public class SampleMetaDataProvider
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
.withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON))
.withFieldList(List.of(
new QFieldMetaData("greetingPrefix", QFieldType.STRING),
new QFieldMetaData("greetingSuffix", QFieldType.STRING)
)))
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
@ -272,9 +339,9 @@ public class SampleMetaDataProvider
{
return new QProcessMetaData()
.withName(PROCESS_NAME_GREET_INTERACTIVE)
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addStep(LoadInitialRecordsStep.defineMetaData("person"))
.addStep(LoadInitialRecordsStep.defineMetaData(TABLE_NAME_PERSON))
.addStep(new QFrontendStepMetaData()
.withName("setup")
@ -289,14 +356,14 @@ public class SampleMetaDataProvider
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
.withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON))
.withFieldList(List.of(
new QFieldMetaData("greetingPrefix", QFieldType.STRING),
new QFieldMetaData("greetingSuffix", QFieldType.STRING)
)))
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName("person")
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))

3
qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java
View File

@ -49,6 +49,8 @@ import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.DisabledOnOs;
import org.junit.jupiter.api.condition.OS;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -57,6 +59,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
**
*******************************************************************************/
@DisabledOnOs(OS.LINUX) // uses database; not available in CI at this time...
class SampleMetaDataProviderTest
{

15
qqq-sample-project/src/test/resources/prime-test-database.sql
View File

@ -29,14 +29,17 @@ CREATE TABLE person
first_name VARCHAR(80) NOT NULL,
last_name VARCHAR(80) NOT NULL,
birth_date DATE,
email VARCHAR(250) NOT NULL
email VARCHAR(250) NOT NULL,
annual_salary DECIMAL(12, 2),
days_worked INTEGER
);
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com');
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com');
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com');
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com');
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com');
INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 75003.50, 1001);
INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 150000, 10100);
INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 300000, 100100);
INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 950000, 75);
INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1500000, 1);
DROP TABLE IF EXISTS carrier;
CREATE TABLE carrier