QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 21:20:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/ONE-69-staging-and-production-go-live' into feature/sprint-8

Browse Source
This commit is contained in:
darin.kelkhoff
2022-08-04 18:37:14 -05:00
parent b5b4ded8aa 3c12979571
commit 6ca6f194b0
40 changed files with 1413 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -31,3 +31,4 @@ target/
hs_err_pid*
.DS_Store
*.swp
.flattened-pom.xml

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java
View File

@ -34,6 +34,9 @@ import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModu
*******************************************************************************/
public class ActionHelper
{
private int f;
/*******************************************************************************
**
@ -42,7 +45,7 @@ public class ActionHelper
{
QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher();
QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(request.getAuthenticationMetaData());
if(!authenticationModule.isSessionValid(request.getSession()))
if(!authenticationModule.isSessionValid(request.getInstance(), request.getSession()))
{
throw new QAuthenticationException("Invalid session in request");
}

82
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java Normal file
View File

@ -0,0 +1,82 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
/*******************************************************************************
** Container wherein backend modules can track data and/or objects that are
** part of a transaction.
**
** Most obvious use-case would be a JDBC Connection. See subclass in rdbms module.
*******************************************************************************/
public class QBackendTransaction
{
/*******************************************************************************
**
*******************************************************************************/
public void commit() throws QException
{
////////////////////////
// noop in base class //
////////////////////////
}
/*******************************************************************************
**
*******************************************************************************/
public void rollback() throws QException
{
////////////////////////
// noop in base class //
////////////////////////
}
/*******************************************************************************
* Closes this stream and releases any system resources associated
* with it. If the stream is already closed then invoking this
* method has no effect.
*
* <p> As noted in {@link AutoCloseable#close()}, cases where the
* close may fail require careful attention. It is strongly advised
* to relinquish the underlying resources and to internally
* <em>mark</em> the {@code Closeable} as closed, prior to throwing
* the {@code IOException}.
*
* @throws IOException
* if an I/O error occurs
*******************************************************************************/
public void close()
{
////////////////////////
// noop in base class //
////////////////////////
}
}

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.actions.interfaces;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@ -37,4 +38,13 @@ public interface InsertInterface
**
*******************************************************************************/
InsertOutput execute(InsertInput insertInput) throws QException;
/*******************************************************************************
**
*******************************************************************************/
default QBackendTransaction openTransaction(InsertInput insertInput) throws QException
{
return (new QBackendTransaction());
}
}

21
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java
View File

@ -25,7 +25,11 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -34,16 +38,25 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
*******************************************************************************/
public class RecordPipe
{
private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(10_000);
private static final Logger LOG = LogManager.getLogger(RecordPipe.class);
private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(1_000);
/*******************************************************************************
** Add a record to the pipe
** Returns true iff the record fit in the pipe; false if the pipe is currently full.
*******************************************************************************/
public boolean addRecord(QRecord record)
public void addRecord(QRecord record)
{
return (queue.offer(record));
boolean offerResult = queue.offer(record);
while(!offerResult)
{
LOG.debug("Record pipe.add failed (due to full pipe). Blocking.");
SleepUtils.sleep(100, TimeUnit.MILLISECONDS);
offerResult = queue.offer(record);
}
}
@ -53,7 +66,7 @@ public class RecordPipe
*******************************************************************************/
public void addRecords(List<QRecord> records)
{
queue.addAll(records);
records.forEach(this::addRecord);
}

30
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@ -47,14 +48,35 @@ public class InsertAction
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend());
QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
// todo pre-customization - just get to modify the request?
InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput);
// todo post-customization - can do whatever w/ the result if you want
return insertOutput;
}
/*******************************************************************************
**
*******************************************************************************/
private QBackendModuleInterface getBackendModuleInterface(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend());
return (qModule);
}
/*******************************************************************************
**
*******************************************************************************/
public QBackendTransaction openTransaction(InsertInput insertInput) throws QException
{
QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
return (qModule.getInsertInterface().openTransaction(insertInput));
}
}

83
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java
View File

@ -28,6 +28,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@ -41,9 +43,44 @@ import org.apache.commons.csv.CSVRecord;
/*******************************************************************************
** Adapter class to convert a CSV string into a list of QRecords.
**
** Based on which method is called, can either take a pipe, and stream records
** into it - or return a list of all records from the file. Either way, at this
** time, the full CSV string is read & parsed - a future optimization might read
** the CSV content from a stream as well.
*******************************************************************************/
public class CsvToQRecordAdapter
{
private RecordPipe recordPipe = null;
private List<QRecord> recordList = null;
/*******************************************************************************
** stream records from a CSV String into a RecordPipe, for a given table, optionally
** using a given mapping.
**
*******************************************************************************/
public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
{
this.recordPipe = recordPipe;
doBuildRecordsFromCsv(csv, table, mapping, recordCustomizer);
}
/*******************************************************************************
** convert a CSV String into a List of QRecords, for a given table, optionally
** using a given mapping.
**
*******************************************************************************/
public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
{
this.recordList = new ArrayList<>();
doBuildRecordsFromCsv(csv, table, mapping, null);
return (recordList);
}
/*******************************************************************************
** convert a CSV String into a List of QRecords, for a given table, optionally
@ -51,14 +88,13 @@ public class CsvToQRecordAdapter
**
** todo - meta-data validation, type handling
*******************************************************************************/
public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
public void doBuildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
{
if(!StringUtils.hasContent(csv))
{
throw (new IllegalArgumentException("Empty csv value was provided."));
}
List<QRecord> rs = new ArrayList<>();
try
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -82,7 +118,7 @@ public class CsvToQRecordAdapter
// put values from the CSV record into a map of header -> value //
//////////////////////////////////////////////////////////////////
Map<String, String> csvValues = new HashMap<>();
for(int i=0; i<headers.size(); i++)
for(int i = 0; i < headers.size(); i++)
{
csvValues.put(headers.get(i), csvRecord.get(i));
}
@ -91,12 +127,14 @@ public class CsvToQRecordAdapter
// now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
QRecord qRecord = new QRecord();
rs.add(qRecord);
for(QFieldMetaData field : table.getFields().values())
{
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
qRecord.setValue(field.getName(), csvValues.get(fieldSource));
}
runRecordCustomizer(recordCustomizer, qRecord);
addRecord(qRecord);
}
}
else if(AbstractQFieldMapping.SourceType.INDEX.equals(mapping.getSourceType()))
@ -125,12 +163,14 @@ public class CsvToQRecordAdapter
// now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
QRecord qRecord = new QRecord();
rs.add(qRecord);
for(QFieldMetaData field : table.getFields().values())
{
Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
qRecord.setValue(field.getName(), csvValues.get(fieldIndex));
}
runRecordCustomizer(recordCustomizer, qRecord);
addRecord(qRecord);
}
}
else
@ -142,8 +182,19 @@ public class CsvToQRecordAdapter
{
throw (new IllegalArgumentException("Error parsing CSV: " + e.getMessage(), e));
}
}
return (rs);
/*******************************************************************************
**
*******************************************************************************/
private void runRecordCustomizer(Consumer<QRecord> recordCustomizer, QRecord qRecord)
{
if(recordCustomizer != null)
{
recordCustomizer.accept(qRecord);
}
}
@ -165,7 +216,7 @@ public class CsvToQRecordAdapter
for(String header : headers)
{
String headerToUse = header;
String headerToUse = header;
String headerWithoutSuffix = header.replaceFirst(" \\d+$", "");
if(countsByHeader.containsKey(headerWithoutSuffix))
@ -183,4 +234,22 @@ public class CsvToQRecordAdapter
return (rs);
}
/*******************************************************************************
** Add a record - either to the pipe, or list, whichever we're building.
*******************************************************************************/
private void addRecord(QRecord record)
{
if(recordPipe != null)
{
recordPipe.addRecord(record);
}
if(recordList != null)
{
recordList.add(record);
}
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java
View File

@ -35,7 +35,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
** Output data container for the RunBackendStep action
**
*******************************************************************************/
public class RunBackendStepOutput extends AbstractActionOutput
public class RunBackendStepOutput extends AbstractActionOutput implements Serializable
{
private ProcessState processState;
private Exception exception; // todo - make optional

35
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
*******************************************************************************/
public class InsertInput extends AbstractTableActionInput
{
private QBackendTransaction transaction;
private List<QRecord> records;
@ -57,6 +59,39 @@ public class InsertInput extends AbstractTableActionInput
/*******************************************************************************
** Getter for transaction
**
*******************************************************************************/
public QBackendTransaction getTransaction()
{
return transaction;
}
/*******************************************************************************
** Setter for transaction
**
*******************************************************************************/
public void setTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
}
/*******************************************************************************
** Fluent setter for transaction
**
*******************************************************************************/
public InsertInput withTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
return (this);
}
/*******************************************************************************
** Getter for records
**

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java
View File

@ -23,10 +23,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -58,35 +56,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
@Override
public void addRecord(QRecord record)
{
if(!recordPipe.addRecord(record))
{
do
{
LOG.debug("Record pipe.add failed (due to full pipe). Blocking.");
SleepUtils.sleep(10, TimeUnit.MILLISECONDS);
}
while(!recordPipe.addRecord(record));
LOG.debug("Done blocking.");
}
}
/*******************************************************************************
**
*******************************************************************************/
private void blockIfPipeIsTooFull()
{
if(recordPipe.countAvailableRecords() >= 100_000)
{
LOG.info("Record pipe is kinda full. Blocking for a bit");
do
{
SleepUtils.sleep(10, TimeUnit.MILLISECONDS);
}
while(recordPipe.countAvailableRecords() >= 10_000);
LOG.info("Done blocking.");
}
recordPipe.addRecord(record);
}
@ -98,7 +68,6 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
public void addRecords(List<QRecord> records)
{
recordPipe.addRecords(records);
blockIfPipeIsTooFull();
}

91
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java
View File

@ -57,20 +57,25 @@ import org.json.JSONObject;
*******************************************************************************/
public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
{
private static final Logger logger = LogManager.getLogger(Auth0AuthenticationModule.class);
private static final Logger LOG = LogManager.getLogger(Auth0AuthenticationModule.class);
private static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 300;
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
// 30 minutes - ideally this would be lower, but right now we've been dealing with re-validation issues... //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
public static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 1800;
public static final String AUTH0_ID_TOKEN_KEY = "sessionId";
public static final String TOKEN_NOT_PROVIDED_ERROR = "Id Token was not provided";
public static final String COULD_NOT_DECODE_ERROR = "Unable to decode id token";
public static final String EXPIRED_TOKEN_ERROR = "Token has expired";
public static final String INVALID_TOKEN_ERROR = "An invalid token was provided";
public static final String COULD_NOT_DECODE_ERROR = "Unable to decode id token";
public static final String EXPIRED_TOKEN_ERROR = "Token has expired";
public static final String INVALID_TOKEN_ERROR = "An invalid token was provided";
private Instant now;
/*******************************************************************************
**
*******************************************************************************/
@ -83,7 +88,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
String idToken = context.get(AUTH0_ID_TOKEN_KEY);
if(idToken == null)
{
logger.warn(TOKEN_NOT_PROVIDED_ERROR);
LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
}
@ -97,7 +102,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// then call method to check more session validity //
/////////////////////////////////////////////////////
QSession qSession = buildQSessionFromToken(idToken);
if(isSessionValid(qSession))
if(isSessionValid(qInstance, qSession))
{
return (qSession);
}
@ -112,7 +117,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// put now into state so we dont check until next interval passes //
///////////////////////////////////////////////////////////////////
StateProviderInterface spi = getStateProvider();
Auth0StateKey key = new Auth0StateKey(qSession.getIdReference());
Auth0StateKey key = new Auth0StateKey(qSession.getIdReference());
spi.put(key, Instant.now());
return (qSession);
@ -122,12 +127,12 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
////////////////////////////////
// could not decode the token //
////////////////////////////////
logger.warn(COULD_NOT_DECODE_ERROR, jde);
LOG.warn(COULD_NOT_DECODE_ERROR, jde);
throw (new QAuthenticationException(COULD_NOT_DECODE_ERROR));
}
catch(TokenExpiredException tee)
{
logger.info(EXPIRED_TOKEN_ERROR, tee);
LOG.info(EXPIRED_TOKEN_ERROR, tee);
throw (new QAuthenticationException(EXPIRED_TOKEN_ERROR));
}
catch(JWTVerificationException | JwkException jve)
@ -135,7 +140,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
///////////////////////////////////////////
// token had invalid signature or claims //
///////////////////////////////////////////
logger.warn(INVALID_TOKEN_ERROR, jve);
LOG.warn(INVALID_TOKEN_ERROR, jve);
throw (new QAuthenticationException(INVALID_TOKEN_ERROR));
}
catch(Exception e)
@ -144,7 +149,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// ¯\_(ツ)_/¯ //
////////////////
String message = "An unknown error occurred";
logger.error(message, e);
LOG.error(message, e);
throw (new QAuthenticationException(message));
}
}
@ -155,16 +160,16 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
**
*******************************************************************************/
@Override
public boolean isSessionValid(QSession session)
public boolean isSessionValid(QInstance instance, QSession session)
{
if(session == null)
{
return (false);
}
StateProviderInterface spi = getStateProvider();
Auth0StateKey key = new Auth0StateKey(session.getIdReference());
Optional<Instant> lastTimeCheckedOptional = spi.get(Instant.class, key);
StateProviderInterface spi = getStateProvider();
Auth0StateKey key = new Auth0StateKey(session.getIdReference());
Optional<Instant> lastTimeCheckedOptional = spi.get(Instant.class, key);
if(lastTimeCheckedOptional.isPresent())
{
Instant lastTimeChecked = lastTimeCheckedOptional.get();
@ -174,7 +179,28 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
// - so this is basically saying, if the time between the last time we checked the token and //
// right now is more than ID_TOKEN_VALIDATION_INTERVAL_SECTIONS, then session needs revalidated //
///////////////////////////////////////////////////////////////////////////////////////////////////
return (Duration.between(lastTimeChecked, Instant.now()).compareTo(Duration.ofSeconds(ID_TOKEN_VALIDATION_INTERVAL_SECONDS)) < 0);
if(Duration.between(lastTimeChecked, Instant.now()).compareTo(Duration.ofSeconds(ID_TOKEN_VALIDATION_INTERVAL_SECONDS)) < 0)
{
return (true);
}
try
{
LOG.debug("Re-validating token due to validation interval being passed: " + session.getIdReference());
revalidateToken(instance, session.getIdReference());
//////////////////////////////////////////////////////////////////
// update the timestamp in state provider, to avoid re-checking //
//////////////////////////////////////////////////////////////////
spi.put(key, Instant.now());
return (true);
}
catch(Exception e)
{
LOG.warn(INVALID_TOKEN_ERROR, e);
return (false);
}
}
return (false);
@ -190,10 +216,10 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
{
Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
DecodedJWT jwt = JWT.decode(idToken);
JwkProvider provider = new UrlJwkProvider(metaData.getBaseUrl());
Jwk jwk = provider.get(jwt.getKeyId());
Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null);
DecodedJWT jwt = JWT.decode(idToken);
JwkProvider provider = new UrlJwkProvider(metaData.getBaseUrl());
Jwk jwk = provider.get(jwt.getKeyId());
Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null);
JWTVerifier verifier = JWT.require(algorithm)
.withIssuer(metaData.getBaseUrl())
.build();
@ -217,20 +243,31 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
////////////////////////////////////
// decode and extract the payload //
////////////////////////////////////
DecodedJWT jwt = JWT.decode(idToken);
Base64.Decoder decoder = Base64.getUrlDecoder();
String payloadString = new String(decoder.decode(jwt.getPayload()));
JSONObject payload = new JSONObject(payloadString);
DecodedJWT jwt = JWT.decode(idToken);
Base64.Decoder decoder = Base64.getUrlDecoder();
String payloadString = new String(decoder.decode(jwt.getPayload()));
JSONObject payload = new JSONObject(payloadString);
QUser qUser = new QUser();
qUser.setFullName(payload.getString("name"));
if(payload.has("name"))
{
qUser.setFullName(payload.getString("name"));
}
else
{
qUser.setFullName("Unknown");
}
if(payload.has("email"))
{
qUser.setIdReference(payload.getString("email"));
}
else
{
qUser.setIdReference(payload.getString("nickname"));
if(payload.has("sub"))
{
qUser.setIdReference(payload.getString("sub"));
}
}
QSession qSession = new QSession();

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java
View File

@ -66,7 +66,7 @@ public class FullyAnonymousAuthenticationModule implements QAuthenticationModule
**
*******************************************************************************/
@Override
public boolean isSessionValid(QSession session)
public boolean isSessionValid(QInstance instance, QSession session)
{
return session != null;
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java
View File

@ -63,7 +63,7 @@ public class MockAuthenticationModule implements QAuthenticationModuleInterface
**
*******************************************************************************/
@Override
public boolean isSessionValid(QSession session)
public boolean isSessionValid(QInstance instance, QSession session)
{
if(session == null)
{

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java
View File

@ -43,5 +43,5 @@ public interface QAuthenticationModuleInterface
/*******************************************************************************
**
*******************************************************************************/
boolean isSessionValid(QSession session);
boolean isSessionValid(QInstance instance, QSession session);
}

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@ -40,6 +41,8 @@ public class BasicETLExtractFunction implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLExtractFunction.class);
private RecordPipe recordPipe = null;
/*******************************************************************************
@ -64,10 +67,35 @@ public class BasicETLExtractFunction implements BackendStep
// queryRequest.setFilter(JsonUtils.toObject(filter, QQueryFilter.class));
// }
//////////////////////////////////////////////////////////////////////
// if the caller gave us a record pipe, pass it to the query action //
//////////////////////////////////////////////////////////////////////
if (recordPipe != null)
{
queryInput.setRecordPipe(recordPipe);
}
QueryAction queryAction = new QueryAction();
QueryOutput queryOutput = queryAction.execute(queryInput);
runBackendStepOutput.setRecords(queryOutput.getRecords());
LOG.info("Query on table " + tableName + " produced " + queryOutput.getRecords().size() + " records.");
if (recordPipe == null)
{
////////////////////////////////////////////////////////////////////////////
// only return the records (and log about them) if there's no record pipe //
////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.setRecords(queryOutput.getRecords());
LOG.info("Query on table " + tableName + " produced " + queryOutput.getRecords().size() + " records.");
}
}
/*******************************************************************************
** Setter for recordPipe
**
*******************************************************************************/
public void setRecordPipe(RecordPipe recordPipe)
{
this.recordPipe = recordPipe;
}
}

17
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -44,6 +45,8 @@ public class BasicETLLoadFunction implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLLoadFunction.class);
private QBackendTransaction transaction;
/*******************************************************************************
@ -86,10 +89,11 @@ public class BasicETLLoadFunction implements BackendStep
insertInput.setSession(runBackendStepInput.getSession());
insertInput.setTableName(table);
insertInput.setRecords(page);
insertInput.setTransaction(transaction);
InsertAction insertAction = new InsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
outputRecords.addAll(insertOutput.getRecords());
// todo - this is to avoid garbage leak in state provider... outputRecords.addAll(insertOutput.getRecords());
recordsInserted += insertOutput.getRecords().size();
}
@ -97,4 +101,15 @@ public class BasicETLLoadFunction implements BackendStep
runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsInserted);
}
/*******************************************************************************
** Setter for transaction
**
*******************************************************************************/
public void setTransaction(QBackendTransaction transaction)
{
this.transaction = transaction;
}
}

240
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java Normal file
View File

@ -0,0 +1,240 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLExtractFunction;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLLoadFunction;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLTransformFunction;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Backend step to do a streamed ETL
*******************************************************************************/
public class StreamedETLBackendStep implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(StreamedETLBackendStep.class);
private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000;
private static final int MAX_SLEEP_MS = 1000;
private static final int INIT_SLEEP_MS = 10;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
QBackendTransaction transaction = openTransaction(runBackendStepInput);
try
{
RecordPipe recordPipe = new RecordPipe();
BasicETLExtractFunction basicETLExtractFunction = new BasicETLExtractFunction();
basicETLExtractFunction.setRecordPipe(recordPipe);
//////////////////////////////////////////
// run the query action as an async job //
//////////////////////////////////////////
AsyncJobManager asyncJobManager = new AsyncJobManager();
String queryJobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) ->
{
basicETLExtractFunction.run(runBackendStepInput, runBackendStepOutput);
return (runBackendStepOutput);
});
LOG.info("Started query job [" + queryJobUUID + "] for report");
AsyncJobState queryJobState = AsyncJobState.RUNNING;
AsyncJobStatus asyncJobStatus = null;
long recordCount = 0;
int nextSleepMillis = INIT_SLEEP_MS;
long lastReceivedRecordsAt = System.currentTimeMillis();
long jobStartTime = System.currentTimeMillis();
while(queryJobState.equals(AsyncJobState.RUNNING))
{
if(recordPipe.countAvailableRecords() == 0)
{
///////////////////////////////////////////////////////////
// if the pipe is empty, sleep to let the producer work. //
// todo - smarter sleep? like get notified vs. sleep? //
///////////////////////////////////////////////////////////
LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work");
SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS);
long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt;
if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS)
{
throw (new QException("Query action appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago)."));
}
}
else
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. //
////////////////////////////////////////////////////////////////////////////////////////////////////////
lastReceivedRecordsAt = System.currentTimeMillis();
nextSleepMillis = INIT_SLEEP_MS;
recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction);
LOG.info(String.format("Processed %,d records so far", recordCount));
}
////////////////////////////////////
// refresh the query job's status //
////////////////////////////////////
Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(queryJobUUID);
if(optionalAsyncJobStatus.isEmpty())
{
/////////////////////////////////////////////////
// todo - ... maybe some version of try-again? //
/////////////////////////////////////////////////
throw (new QException("Could not get status of report query job [" + queryJobUUID + "]"));
}
asyncJobStatus = optionalAsyncJobStatus.get();
queryJobState = asyncJobStatus.getState();
}
LOG.info("Query job [" + queryJobUUID + "] for ETL completed with status: " + asyncJobStatus);
//////////////////////////////////////////////////////
// send the final records to transform & load steps //
//////////////////////////////////////////////////////
recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction);
/////////////////////
// commit the work //
/////////////////////
transaction.commit();
long reportEndTime = System.currentTimeMillis();
LOG.info(String.format("Processed %,d records", recordCount)
+ String.format(" at end of ETL job in %,d ms (%.2f records/second).", (reportEndTime - jobStartTime), 1000d * (recordCount / (.001d + (reportEndTime - jobStartTime)))));
runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount);
}
catch(Exception e)
{
////////////////////////////////////////////////////////////////////////////////
// rollback the work, then re-throw the error for up-stream to catch & report //
////////////////////////////////////////////////////////////////////////////////
transaction.rollback();
throw (e);
}
finally
{
////////////////////////////////////////////////////////////
// always close our transactions (e.g., jdbc connections) //
////////////////////////////////////////////////////////////
transaction.close();
}
}
/*******************************************************************************
**
*******************************************************************************/
private QBackendTransaction openTransaction(RunBackendStepInput runBackendStepInput) throws QException
{
InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance());
insertInput.setSession(runBackendStepInput.getSession());
insertInput.setTableName(runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE));
return new InsertAction().openTransaction(insertInput);
}
/*******************************************************************************
**
*******************************************************************************/
private int consumeRecordsFromPipe(RecordPipe recordPipe, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, QBackendTransaction transaction) throws QException
{
List<QRecord> qRecords = recordPipe.consumeAvailableRecords();
preTransform(qRecords, runBackendStepInput, runBackendStepOutput);
runBackendStepInput.setRecords(qRecords);
new BasicETLTransformFunction().run(runBackendStepInput, runBackendStepOutput);
postTransform(qRecords, runBackendStepInput, runBackendStepOutput);
runBackendStepInput.setRecords(runBackendStepOutput.getRecords());
BasicETLLoadFunction basicETLLoadFunction = new BasicETLLoadFunction();
basicETLLoadFunction.setTransaction(transaction);
basicETLLoadFunction.run(runBackendStepInput, runBackendStepOutput);
return (qRecords.size());
}
/*******************************************************************************
** Customization point for subclasses of this step.
*******************************************************************************/
protected void preTransform(List<QRecord> qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
////////////////////////
// noop in base class //
////////////////////////
}
/*******************************************************************************
** Customization point for subclasses of this step.
*******************************************************************************/
protected void postTransform(List<QRecord> qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
////////////////////////
// noop in base class //
////////////////////////
}
}

75
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java Normal file
View File

@ -0,0 +1,75 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
/*******************************************************************************
** Definition for Streamed ETL process.
*******************************************************************************/
public class StreamedETLProcess
{
public static final String PROCESS_NAME = "etl.streamed";
public static final String FUNCTION_NAME_ETL = "streamedETL";
public static final String FIELD_SOURCE_TABLE = "sourceTable";
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
public static final String FIELD_MAPPING_JSON = "mappingJSON";
public static final String FIELD_RECORD_COUNT = "recordCount";
/*******************************************************************************
**
*******************************************************************************/
public QProcessMetaData defineProcessMetaData()
{
QStepMetaData etlFunction = new QBackendStepMetaData()
.withName(FUNCTION_NAME_ETL)
.withCode(new QCodeReference()
.withName(StreamedETLBackendStep.class.getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
.withField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING))
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)))
.withOutputMetaData(new QFunctionOutputMetaData()
.addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));
return new QProcessMetaData()
.withName(PROCESS_NAME)
.addStep(etlFunction);
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java
View File

@ -80,7 +80,7 @@ class ReportActionTest
public void testBigger() throws Exception
{
// int recordCount = 2_000_000; // to really stress locally, use this.
int recordCount = 200_000;
int recordCount = 50_000;
String filename = "/tmp/ReportActionTest.csv";
runReport(recordCount, filename, ReportFormat.CSV, false);

89
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.modules.authentication;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
@ -31,7 +32,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.Auth0AuthenticationMetaData;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.AUTH0_ID_TOKEN_KEY;
@ -40,7 +40,8 @@ import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0Authent
import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.INVALID_TOKEN_ERROR;
import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.TOKEN_NOT_PROVIDED_ERROR;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@ -49,9 +50,9 @@ import static org.junit.jupiter.api.Assertions.fail;
*******************************************************************************/
public class Auth0AuthenticationModuleTest
{
private static final String VALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA";
private static final String INVALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA-thismakesinvalid";
private static final String EXPIRED_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE4VDIxOjM4OjE1LjM4NloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MTgwNDc3LCJleHAiOjE2NTgyMTY0NzcsIm5vbmNlIjoiVkZkQlYzWmplR2hvY1cwMk9WZEtabHBLU0c1K1ZXbElhMEV3VkZaeFpVdEJVMDErZUZaT1RtMTNiZz09In0.fU7EwUgNrupOPz_PX_aQKON2xG1-LWD85xVo1Bn41WNEek-iMyJoch8l6NUihi7Bou14BoOfeWIG_sMqsLHqI2Pk7el7l1kigsjURx0wpiXadBt8piMxdIlxdToZEMuZCBzg7eJvXh4sM8tlV5cm0gPa6FT9Ih3VGJajNlXi5BcYS_JRpIvFvHn8-Bxj4KiAlZ5XPPkopjnDgP8kFfc4cMn_nxDkqWYlhj-5TaGW2xCLC9Qr_9UNxX0fm-CkKjYs3Z5ezbiXNkc-bxrCYvxeBeDPf8-T3EqrxCRVqCZSJ85BHdOc_E7UZC_g8bNj0umoplGwlCbzO4XIuOO-KlIaOg";
private static final String VALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA";
private static final String INVALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA-thismakesinvalid";
private static final String EXPIRED_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE4VDIxOjM4OjE1LjM4NloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MTgwNDc3LCJleHAiOjE2NTgyMTY0NzcsIm5vbmNlIjoiVkZkQlYzWmplR2hvY1cwMk9WZEtabHBLU0c1K1ZXbElhMEV3VkZaeFpVdEJVMDErZUZaT1RtMTNiZz09In0.fU7EwUgNrupOPz_PX_aQKON2xG1-LWD85xVo1Bn41WNEek-iMyJoch8l6NUihi7Bou14BoOfeWIG_sMqsLHqI2Pk7el7l1kigsjURx0wpiXadBt8piMxdIlxdToZEMuZCBzg7eJvXh4sM8tlV5cm0gPa6FT9Ih3VGJajNlXi5BcYS_JRpIvFvHn8-Bxj4KiAlZ5XPPkopjnDgP8kFfc4cMn_nxDkqWYlhj-5TaGW2xCLC9Qr_9UNxX0fm-CkKjYs3Z5ezbiXNkc-bxrCYvxeBeDPf8-T3EqrxCRVqCZSJ85BHdOc_E7UZC_g8bNj0umoplGwlCbzO4XIuOO-KlIaOg";
private static final String UNDECODABLE_TOKEN = "UNDECODABLE";
public static final String AUTH0_BASE_URL = "https://kingsrook.us.auth0.com/";
@ -59,32 +60,79 @@ public class Auth0AuthenticationModuleTest
/*******************************************************************************
** Test a valid token where 'now' is set to a time that would be valid for it
** Test an expired token where 'now' is set to a time that would not require it to be
** re-checked, so it'll show as valid
**
*******************************************************************************/
@Test
public void testLastTimeChecked() throws QAuthenticationException
public void testLastTimeCheckedNow()
{
//////////////////////////////////////////////////////////
// Tuesday, July 19, 2022 12:40:27.299 PM GMT-05:00 DST //
//////////////////////////////////////////////////////////
Instant now = Instant.now();
assertTrue(testLastTimeChecked(Instant.now(), UNDECODABLE_TOKEN), "A session just checked 'now' should always be valid");
}
/////////////////////////////////////////////////////////
// put the 'now' from the past into the state provider //
/////////////////////////////////////////////////////////
StateProviderInterface spi = InMemoryStateProvider.getInstance();
Auth0AuthenticationModule.Auth0StateKey key = new Auth0AuthenticationModule.Auth0StateKey(VALID_TOKEN);
spi.put(key, now);
/*******************************************************************************
** Test an expired token where 'now' is set to a time that would not require it to be
** re-checked, so it'll show as valid
**
*******************************************************************************/
@Test
public void testLastTimeCheckedJustUnderThreshold()
{
Instant underThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS - 60, ChronoUnit.SECONDS);
assertTrue(testLastTimeChecked(underThreshold, INVALID_TOKEN), "A session checked under threshold should be valid");
}
/*******************************************************************************
** Test an expired token where 'now' is set to a time that would require it to be
** re-checked
**
*******************************************************************************/
@Test
public void testLastTimeCheckedJustOverThreshold()
{
Instant overThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS + 60, ChronoUnit.SECONDS);
assertFalse(testLastTimeChecked(overThreshold, INVALID_TOKEN), "A session checked over threshold should be re-validated, and in this case, not be valid.");
}
/*******************************************************************************
** Test an expired token where 'now' is set to a time that would require it to be
** re-checked
**
*******************************************************************************/
@Test
public void testLastTimeCheckedOverThresholdAndUndecodable()
{
Instant overThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS + 60, ChronoUnit.SECONDS);
assertFalse(testLastTimeChecked(overThreshold, UNDECODABLE_TOKEN), "A session checked over threshold should be re-validated, and in this case, not be valid.");
}
/*******************************************************************************
**
*******************************************************************************/
private boolean testLastTimeChecked(Instant lastTimeChecked, String token)
{
/////////////////////////////////////////////////////////////
// put the input last-time-checked into the state provider //
/////////////////////////////////////////////////////////////
Auth0AuthenticationModule.Auth0StateKey key = new Auth0AuthenticationModule.Auth0StateKey(token);
InMemoryStateProvider.getInstance().put(key, lastTimeChecked);
//////////////////////
// build up session //
//////////////////////
QSession session = new QSession();
session.setIdReference(VALID_TOKEN);
session.setIdReference(token);
Auth0AuthenticationModule auth0AuthenticationModule = new Auth0AuthenticationModule();
assertEquals(true, auth0AuthenticationModule.isSessionValid(session), "Session should return as still valid.");
return (auth0AuthenticationModule.isSessionValid(getQInstance(), session));
}
@ -114,7 +162,7 @@ public class Auth0AuthenticationModuleTest
/*******************************************************************************
** Test failure case, token cant be decoded
** Test failure case, token can't be decoded
**
*******************************************************************************/
@Test
@ -220,4 +268,5 @@ public class Auth0AuthenticationModuleTest
qInstance.setAuthentication(authenticationMetaData);
return (qInstance);
}
}

4
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java
View File

@ -49,8 +49,8 @@ public class FullyAnonymousAuthenticationModuleTest
assertNotNull(session.getIdReference(), "Session id ref should not be null");
assertNotNull(session.getUser(), "Session User should not be null");
assertNotNull(session.getUser().getIdReference(), "Session User id ref should not be null");
assertTrue(fullyAnonymousAuthenticationModule.isSessionValid(session), "Any session should be valid");
assertFalse(fullyAnonymousAuthenticationModule.isSessionValid(null), "null should be not valid");
assertTrue(fullyAnonymousAuthenticationModule.isSessionValid(null, session), "Any session should be valid");
assertFalse(fullyAnonymousAuthenticationModule.isSessionValid(null, null), "null should be not valid");
}
}

89
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java Normal file
View File

@ -0,0 +1,89 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLProcess
*******************************************************************************/
class StreamedETLProcessTest
{
/*******************************************************************************
** Simplest happy path
*******************************************************************************/
@Test
public void test() throws QException
{
RunProcessInput request = new RunProcessInput(TestUtils.defineInstance());
request.setSession(TestUtils.getMockSession());
request.setProcessName(StreamedETLProcess.PROCESS_NAME);
request.addValue(StreamedETLProcess.FIELD_SOURCE_TABLE, TestUtils.defineTablePerson().getName());
request.addValue(StreamedETLProcess.FIELD_DESTINATION_TABLE, TestUtils.definePersonFileTable().getName());
request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, "");
RunProcessOutput result = new RunProcessAction().execute(request);
assertNotNull(result);
assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
assertTrue(result.getException().isEmpty());
}
/*******************************************************************************
** Basic example of doing a mapping transformation
*******************************************************************************/
@Test
public void testMappingTransformation() throws QException
{
RunProcessInput request = new RunProcessInput(TestUtils.defineInstance());
request.setSession(TestUtils.getMockSession());
request.setProcessName(StreamedETLProcess.PROCESS_NAME);
request.addValue(StreamedETLProcess.FIELD_SOURCE_TABLE, TestUtils.definePersonFileTable().getName());
request.addValue(StreamedETLProcess.FIELD_DESTINATION_TABLE, TestUtils.defineTableIdAndNameOnly().getName());
///////////////////////////////////////////////////////////////////////////////////////
// define our mapping from destination-table field names to source-table field names //
///////////////////////////////////////////////////////////////////////////////////////
QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping().withMapping("name", "firstName");
// request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping.getMapping()));
request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping));
RunProcessOutput result = new RunProcessAction().execute(request);
assertNotNull(result);
assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
assertTrue(result.getException().isEmpty());
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthentic
import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
/*******************************************************************************
@ -91,6 +92,7 @@ public class TestUtils
qInstance.addProcess(defineProcessGreetPeopleInteractive());
qInstance.addProcess(defineProcessAddToPeoplesAge());
qInstance.addProcess(new BasicETLProcess().defineProcessMetaData());
qInstance.addProcess(new StreamedETLProcess().defineProcessMetaData());
System.out.println(new QInstanceAdapter().qInstanceToJson(qInstance));

30
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -201,10 +201,16 @@ public abstract class AbstractBaseFilesystemAction<FILE>
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
if(queryInput.getRecordPipe() != null)
{
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> addBackendDetailsToRecord(record, file)));
}
else
{
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
}
break;
}
case JSON:
@ -212,6 +218,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
// todo - pipe support!!
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
@ -241,10 +248,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/
protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, FILE file)
{
recordsInFile.forEach(record ->
{
record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file));
});
recordsInFile.forEach(r -> addBackendDetailsToRecord(r, file));
}
/*******************************************************************************
** Add backend details to a record about the file that it is in.
*******************************************************************************/
protected void addBackendDetailsToRecord(QRecord record, FILE file)
{
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file));
}

2
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java
View File

@ -94,11 +94,13 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep
String moveOrDelete = runBackendStepInput.getValueString(FIELD_MOVE_OR_DELETE);
if(VALUE_DELETE.equals(moveOrDelete))
{
LOG.info("Deleting ETL source file: " + sourceFile);
actionBase.deleteFile(runBackendStepInput.getInstance(), table, sourceFile);
}
else if(VALUE_MOVE.equals(moveOrDelete))
{
String destinationForMoves = runBackendStepInput.getValueString(FIELD_DESTINATION_FOR_MOVES);
LOG.info("Moving ETL source file: " + sourceFile + " to " + destinationForMoves);
if(!StringUtils.hasContent(destinationForMoves))
{
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));

75
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStep.java Normal file
View File

@ -0,0 +1,75 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLBackendStep;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic.BasicETLCollectSourceFileNamesStep;
/*******************************************************************************
** Extension to the base StreamedETLBackendStep, unique for where the source
** table is a filesystem, where we want/need to collect the filenames that were
** processed in the Extract step, so they can be passed into the cleanup step.
**
** Similar in purpose to the BasicETLCollectSourceFileNamesStep - only in this
** case, due to the streaming behavior of the StreamedETLProcess, we can't really
** inject this code as a separate backend step - so instead we extend that step,
** and override its postTransform method to intercept the records & file names.
*******************************************************************************/
public class StreamedETLFilesystemBackendStep extends StreamedETLBackendStep
{
/*******************************************************************************
**
*******************************************************************************/
@Override
protected void preTransform(List<QRecord> qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
Set<String> sourceFiles = qRecords.stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet());
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// expect that we'll be called on multiple "pages" of records as they run through the pipe. //
// each time we're called, we need to: //
// - get the unique file paths in this list of records //
// - if we previously set the list of file names in the output, then split that value up and add those names to the set we see now //
// - set the list of name (joined by commas) in the output //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
String existingListOfFileNames = runBackendStepOutput.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS);
if(existingListOfFileNames != null)
{
sourceFiles.addAll(List.of(existingListOfFileNames.split(",")));
}
runBackendStepOutput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
}
}

4
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java
View File

@ -137,7 +137,7 @@ public class S3Utils
////////////////////////////////////////////////////////////////////////////////
if(key.endsWith("/"))
{
LOG.debug("Skipping file [{}] because it is a folder", key);
// LOG.debug("Skipping file [{}] because it is a folder", key);
continue;
}
@ -146,7 +146,7 @@ public class S3Utils
///////////////////////////////////////////
if(!pathMatcher.matches(Path.of(URI.create("file:///" + key))))
{
LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
// LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
continue;
}

134
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -24,20 +24,27 @@ package com.kingsrook.qqq.backend.module.filesystem;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed.StreamedETLFilesystemBackendStep;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
@ -49,10 +56,16 @@ import org.apache.commons.io.FileUtils;
*******************************************************************************/
public class TestUtils
{
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String TABLE_NAME_PERSON_LOCAL_FS = "person";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
///////////////////////////////////////////////////////////////////
// shouldn't be accessed directly, as we append a counter to it. //
@ -112,14 +125,18 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QInstance defineInstance() throws QInstanceValidationException
public static QInstance defineInstance() throws QException
{
QInstance qInstance = new QInstance();
qInstance.setAuthentication(defineAuthentication());
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addBackend(defineS3Backend());
qInstance.addTable(defineS3CSVPersonTable());
qInstance.addBackend(defineMockBackend());
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
new QInstanceValidator().validate(qInstance);
@ -159,21 +176,55 @@ public class TestUtils
public static QTableMetaData defineLocalFilesystemJSONPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_LOCAL_FS)
.withName(TABLE_NAME_PERSON_LOCAL_FS_JSON)
.withLabel("Person")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withFields(defineCommonPersonTableFields())
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("persons")
.withRecordFormat(RecordFormat.JSON)
.withCardinality(Cardinality.MANY)
.withGlob("*.json")
);
}
/*******************************************************************************
**
*******************************************************************************/
private static List<QFieldMetaData> defineCommonPersonTableFields()
{
return (List.of(
new QFieldMetaData("id", QFieldType.INTEGER),
new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"),
new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"),
new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"),
new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"),
new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"),
new QFieldMetaData("email", QFieldType.STRING)
));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemCSVPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_LOCAL_FS_CSV)
.withLabel("Person")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("id")
.withFields(defineCommonPersonTableFields())
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("persons-csv")
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
.withGlob("*.csv")
);
}
@ -202,13 +253,7 @@ public class TestUtils
.withLabel("Person S3 Table")
.withBackendName(defineS3Backend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withFields(defineCommonPersonTableFields())
.withBackendDetails(new S3TableBackendDetails()
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
@ -220,7 +265,52 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QSession getMockSession() throws QInstanceValidationException
public static QBackendMetaData defineMockBackend()
{
return (new QBackendMetaData()
.withBackendType("mock")
.withName(BACKEND_NAME_MOCK));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineMockPersonTable()
{
return (new QTableMetaData()
.withName(TABLE_NAME_PERSON_MOCK)
.withLabel("Person Mock Table")
.withBackendName(BACKEND_NAME_MOCK)
.withPrimaryKeyField("id")
.withFields(defineCommonPersonTableFields()));
}
/*******************************************************************************
**
*******************************************************************************/
private static QProcessMetaData defineStreamedLocalCsvToMockETLProcess() throws QException
{
QProcessMetaData qProcessMetaData = new StreamedETLProcess().defineProcessMetaData();
qProcessMetaData.setName(PROCESS_NAME_STREAMED_ETL);
QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
backendStep.setCode(new QCodeReference(StreamedETLFilesystemBackendStep.class));
backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_SOURCE_TABLE).setDefaultValue(TABLE_NAME_PERSON_LOCAL_FS_CSV);
backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_DESTINATION_TABLE).setDefaultValue(TABLE_NAME_PERSON_MOCK);
return (qProcessMetaData);
}
/*******************************************************************************
**
*******************************************************************************/
public static QSession getMockSession() throws QException
{
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
return (mockAuthenticationModule.createSession(defineInstance(), null));

8
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java
View File

@ -70,7 +70,7 @@ public class FilesystemBackendModuleTest
public void testDeleteFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
@ -94,7 +94,7 @@ public class FilesystemBackendModuleTest
public void testDeleteFileDoesNotExist() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
@ -120,7 +120,7 @@ public class FilesystemBackendModuleTest
public void testMoveFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
@ -157,7 +157,7 @@ public class FilesystemBackendModuleTest
public void testMoveFileDoesNotExit() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));

37
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java
View File

@ -80,7 +80,8 @@ public class FilesystemActionTest
fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module");
}
writePersonFiles(baseDirectory);
writePersonJSONFiles(baseDirectory);
writePersonCSVFiles(baseDirectory);
}
@ -88,7 +89,7 @@ public class FilesystemActionTest
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writePersonFiles(File baseDirectory) throws IOException
private void writePersonJSONFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
@ -118,6 +119,38 @@ public class FilesystemActionTest
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writePersonCSVFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
{
if (StringUtils.hasContent(details.getBasePath()))
{
fullPath += File.separatorChar + details.getBasePath();
}
}
fullPath += File.separatorChar;
String csvData1 = """
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
"1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
"2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
""";
FileUtils.writeStringToFile(new File(fullPath + "FILE-1.csv"), csvData1);
String csvData2 = """
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
"3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
"4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
"5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com\""""; // intentionally no \n at EOL here
FileUtils.writeStringToFile(new File(fullPath + "FILE-2.csv"), csvData2);
}
/*******************************************************************************
**
*******************************************************************************/

2
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java
View File

@ -71,7 +71,7 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
QueryInput queryInput = new QueryInput();
QInstance instance = TestUtils.defineInstance();
QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference()
.withName(ValueUpshifter.class.getName())
.withCodeType(QCodeType.JAVA)

10
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java
View File

@ -24,7 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -44,7 +44,7 @@ class FilesystemBackendMetaDataTest
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
public void testSerializingToJson() throws QException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
@ -62,7 +62,7 @@ class FilesystemBackendMetaDataTest
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
public void testDeserializingFromJson() throws IOException, QException
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
@ -71,6 +71,8 @@ class FilesystemBackendMetaDataTest
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized.getBackends()).usingRecursiveComparison()
// TODO seeing occassional flaps on this field - where it can be null 1 out of 10 runs... unclear why.
.ignoringFields("mock.backendType")
.isEqualTo(qInstance.getBackends());
}
}
}

4
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java
View File

@ -198,7 +198,7 @@ public class BasicETLCleanupSourceFilesStepTest
runBackendStepInput.setProcessName(qProcessMetaData.getName());
// runFunctionRequest.setRecords(records);
runBackendStepInput.setSession(TestUtils.getMockSession());
runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
runBackendStepInput.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runBackendStepInput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
@ -219,7 +219,7 @@ public class BasicETLCleanupSourceFilesStepTest
private String getRandomFilePathPersonTable(QInstance qInstance)
{
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails();
FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON).getBackendDetails();
String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath();
String filePath = tablePath + File.separator + UUID.randomUUID();
return filePath;

62
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStepTest.java Normal file
View File

@ -0,0 +1,62 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic.BasicETLCollectSourceFileNamesStep;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Unit test for StreamedETLFilesystemBackendStep
*******************************************************************************/
class StreamedETLFilesystemBackendStepTest extends FilesystemActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void runFullProcess() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(TestUtils.getMockSession());
runProcessInput.setProcessName(TestUtils.PROCESS_NAME_STREAMED_ETL);
RunProcessOutput output = new RunProcessAction().execute(runProcessInput);
String sourceFilePaths = ValueUtils.getValueAsString(output.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS));
assertThat(sourceFilePaths)
.contains("FILE-1.csv")
.contains("FILE-2.csv");
}
}

3
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java
View File

@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -60,7 +59,7 @@ public class S3QueryActionTest extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
private QueryInput initQueryRequest() throws QInstanceValidationException
private QueryInput initQueryRequest() throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setInstance(TestUtils.defineInstance());

11
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java
View File

@ -22,9 +22,8 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -44,7 +43,7 @@ class S3BackendMetaDataTest
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
public void testSerializingToJson() throws QException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
@ -62,7 +61,7 @@ class S3BackendMetaDataTest
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
public void testDeserializingFromJson() throws Exception
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
@ -71,6 +70,8 @@ class S3BackendMetaDataTest
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized.getBackends()).usingRecursiveComparison()
// TODO seeing occassional flaps on this field - where it can be null 1 out of 10 runs... unclear why.
.ignoringFields("mock.backendType")
.isEqualTo(qInstance.getBackends());
}
}
}

47
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
View File

@ -28,6 +28,7 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@ -92,7 +93,20 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
List<QRecord> outputRecords = new ArrayList<>();
rs.setRecords(outputRecords);
try(Connection connection = getConnection(insertInput))
Connection connection;
boolean needToCloseConnection = false;
if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
{
LOG.debug("Using connection from insertInput [" + rdbmsTransaction.getConnection() + "]");
connection = rdbmsTransaction.getConnection();
}
else
{
connection = getConnection(insertInput);
needToCloseConnection = true;
}
try
{
for(List<QRecord> page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE))
{
@ -130,6 +144,13 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
}
}
}
finally
{
if(needToCloseConnection)
{
connection.close();
}
}
return rs;
}
@ -139,4 +160,28 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QBackendTransaction openTransaction(InsertInput insertInput) throws QException
{
try
{
LOG.info("Opening transaction");
Connection connection = getConnection(insertInput);
connection.setAutoCommit(false);
System.out.println("Set connection [" + connection + "] to auto-commit false");
return (new RDBMSTransaction(connection));
}
catch(Exception e)
{
throw new QException("Error opening transaction: " + e.getMessage(), e);
}
}
}

141
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java Normal file
View File

@ -0,0 +1,141 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.rdbms.actions;
import java.io.IOException;
import java.sql.Connection;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** RDBMS implementation of backend transaction.
**
** Stores a jdbc connection, which is set to autoCommit(false).
*******************************************************************************/
public class RDBMSTransaction extends QBackendTransaction
{
private static final Logger LOG = LogManager.getLogger(RDBMSTransaction.class);
private Connection connection;
/*******************************************************************************
**
*******************************************************************************/
public RDBMSTransaction(Connection connection)
{
this.connection = connection;
}
/*******************************************************************************
** Getter for connection
**
*******************************************************************************/
public Connection getConnection()
{
return connection;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void commit() throws QException
{
try
{
RDBMSTransaction.LOG.info("Committing transaction");
System.out.println("Calling commit on connection [" + connection + "]");
connection.commit();
RDBMSTransaction.LOG.info("Commit complete");
}
catch(Exception e)
{
RDBMSTransaction.LOG.error("Error committing transaction", e);
throw new QException("Error committing transaction: " + e.getMessage(), e);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void rollback() throws QException
{
try
{
RDBMSTransaction.LOG.info("Rolling back transaction");
connection.rollback();
RDBMSTransaction.LOG.info("Rollback complete");
}
catch(Exception e)
{
RDBMSTransaction.LOG.error("Error rolling back transaction", e);
throw new QException("Error rolling back transaction: " + e.getMessage(), e);
}
}
/*******************************************************************************
* Closes this stream and releases any system resources associated
* with it. If the stream is already closed then invoking this
* method has no effect.
*
* <p> As noted in {@link AutoCloseable#close()}, cases where the
* close may fail require careful attention. It is strongly advised
* to relinquish the underlying resources and to internally
* <em>mark</em> the {@code Closeable} as closed, prior to throwing
* the {@code IOException}.
*
* @throws IOException
* if an I/O error occurs
*******************************************************************************/
@Override
public void close()
{
try
{
if(connection.isClosed())
{
return;
}
connection.close();
}
catch(Exception e)
{
LOG.error("Error closing connection - possible jdbc connection leak", e);
}
}
}

22
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java
View File

@ -690,6 +690,11 @@ public class QueryManager
bindParam(statement, index, l.intValue());
return (1);
}
else if(value instanceof Double d)
{
bindParam(statement, index, d.doubleValue());
return (1);
}
else if(value instanceof String s)
{
bindParam(statement, index, s);
@ -860,6 +865,23 @@ public class QueryManager
/*******************************************************************************
*
*******************************************************************************/
public static void bindParam(PreparedStatement statement, int index, Double value) throws SQLException
{
if(value == null)
{
statement.setNull(index, Types.DOUBLE);
}
else
{
statement.setDouble(index, value);
}
}
/*******************************************************************************
*
*******************************************************************************/

32
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -251,11 +251,39 @@ public class QJavalinImplementation
try
{
Map<String, String> authenticationContext = new HashMap<>();
authenticationContext.put(SESSION_ID_COOKIE_NAME, context.cookie(SESSION_ID_COOKIE_NAME));
/////////////////////////////////////////////////////////////////////////////////
// look for a token in either the sessionId cookie, or an Authorization header //
/////////////////////////////////////////////////////////////////////////////////
String sessionIdCookieValue = context.cookie(SESSION_ID_COOKIE_NAME);
if(StringUtils.hasContent(sessionIdCookieValue))
{
authenticationContext.put(SESSION_ID_COOKIE_NAME, sessionIdCookieValue);
}
else
{
String authorizationHeaderValue = context.header("Authorization");
if (authorizationHeaderValue != null)
{
String bearerPrefix = "Bearer ";
if(authorizationHeaderValue.startsWith(bearerPrefix))
{
authorizationHeaderValue = authorizationHeaderValue.replaceFirst(bearerPrefix, "");
}
authenticationContext.put(SESSION_ID_COOKIE_NAME, authorizationHeaderValue);
}
}
QSession session = authenticationModule.createSession(qInstance, authenticationContext);
input.setSession(session);
context.cookie(SESSION_ID_COOKIE_NAME, session.getIdReference(), SESSION_COOKIE_AGE);
/////////////////////////////////////////////////////////////////////////////////
// if we got a session id cookie in, then send it back with updated cookie age //
/////////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(sessionIdCookieValue))
{
context.cookie(SESSION_ID_COOKIE_NAME, session.getIdReference(), SESSION_COOKIE_AGE);
}
}
catch(QAuthenticationException qae)
{