diff --git a/.gitignore b/.gitignore index 64c6c21c..e6a143b9 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ target/ hs_err_pid* .DS_Store *.swp +.flattened-pom.xml diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java index d5ab7318..11ef0eeb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java @@ -34,6 +34,9 @@ import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModu *******************************************************************************/ public class ActionHelper { + private int f; + + /******************************************************************************* ** @@ -42,7 +45,7 @@ public class ActionHelper { QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher(); QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(request.getAuthenticationMetaData()); - if(!authenticationModule.isSessionValid(request.getSession())) + if(!authenticationModule.isSessionValid(request.getInstance(), request.getSession())) { throw new QAuthenticationException("Invalid session in request"); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java new file mode 100644 index 00000000..4220ec90 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java @@ -0,0 +1,82 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions; + + +import java.io.IOException; +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** Container wherein backend modules can track data and/or objects that are + ** part of a transaction. + ** + ** Most obvious use-case would be a JDBC Connection. See subclass in rdbms module. + *******************************************************************************/ +public class QBackendTransaction +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public void commit() throws QException + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void rollback() throws QException + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + + /******************************************************************************* + * Closes this stream and releases any system resources associated + * with it. If the stream is already closed then invoking this + * method has no effect. + * + *

As noted in {@link AutoCloseable#close()}, cases where the + * close may fail require careful attention. It is strongly advised + * to relinquish the underlying resources and to internally + * mark the {@code Closeable} as closed, prior to throwing + * the {@code IOException}. + * + * @throws IOException + * if an I/O error occurs + *******************************************************************************/ + public void close() + { + //////////////////////// + // noop in base class // + //////////////////////// + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java index 34e79045..dc90de66 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.actions.interfaces; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; @@ -37,4 +38,13 @@ public interface InsertInterface ** *******************************************************************************/ InsertOutput execute(InsertInput insertInput) throws QException; + + /******************************************************************************* + ** + *******************************************************************************/ + default QBackendTransaction openTransaction(InsertInput insertInput) throws QException + { + return (new QBackendTransaction()); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java index cb8e3b6d..3703a841 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -25,7 +25,11 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -34,16 +38,25 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; *******************************************************************************/ public class RecordPipe { - private ArrayBlockingQueue queue = new ArrayBlockingQueue<>(10_000); + private static final Logger LOG = LogManager.getLogger(RecordPipe.class); + + private ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1_000); /******************************************************************************* ** Add a record to the pipe ** Returns true iff the record fit in the pipe; false if the pipe is currently full. *******************************************************************************/ - public boolean addRecord(QRecord record) + public void addRecord(QRecord record) { - return (queue.offer(record)); + boolean offerResult = queue.offer(record); + + while(!offerResult) + { + LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); + SleepUtils.sleep(100, TimeUnit.MILLISECONDS); + offerResult = queue.offer(record); + } } @@ -53,7 +66,7 @@ public class RecordPipe *******************************************************************************/ public void addRecords(List records) { - queue.addAll(records); + records.forEach(this::addRecord); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index d4f6cd92..54ebcfb2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; @@ -47,14 +48,35 @@ public class InsertAction *******************************************************************************/ public InsertOutput execute(InsertInput insertInput) throws QException { - ActionHelper.validateSession(insertInput); - - QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); - QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend()); + QBackendModuleInterface qModule = getBackendModuleInterface(insertInput); // todo pre-customization - just get to modify the request? InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput); // todo post-customization - can do whatever w/ the result if you want return insertOutput; } + + + /******************************************************************************* + ** + *******************************************************************************/ + private QBackendModuleInterface getBackendModuleInterface(InsertInput insertInput) throws QException + { + ActionHelper.validateSession(insertInput); + QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); + QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend()); + return (qModule); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QBackendTransaction openTransaction(InsertInput insertInput) throws QException + { + QBackendModuleInterface qModule = getBackendModuleInterface(insertInput); + return (qModule.getInsertInterface().openTransaction(insertInput)); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java index 38b7f58a..3d5493a3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -28,6 +28,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -41,9 +43,44 @@ import org.apache.commons.csv.CSVRecord; /******************************************************************************* ** Adapter class to convert a CSV string into a list of QRecords. ** + ** Based on which method is called, can either take a pipe, and stream records + ** into it - or return a list of all records from the file. Either way, at this + ** time, the full CSV string is read & parsed - a future optimization might read + ** the CSV content from a stream as well. *******************************************************************************/ public class CsvToQRecordAdapter { + private RecordPipe recordPipe = null; + private List recordList = null; + + + + /******************************************************************************* + ** stream records from a CSV String into a RecordPipe, for a given table, optionally + ** using a given mapping. + ** + *******************************************************************************/ + public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping mapping, Consumer recordCustomizer) + { + this.recordPipe = recordPipe; + doBuildRecordsFromCsv(csv, table, mapping, recordCustomizer); + } + + + + /******************************************************************************* + ** convert a CSV String into a List of QRecords, for a given table, optionally + ** using a given mapping. + ** + *******************************************************************************/ + public List buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping) + { + this.recordList = new ArrayList<>(); + doBuildRecordsFromCsv(csv, table, mapping, null); + return (recordList); + } + + /******************************************************************************* ** convert a CSV String into a List of QRecords, for a given table, optionally @@ -51,14 +88,13 @@ public class CsvToQRecordAdapter ** ** todo - meta-data validation, type handling *******************************************************************************/ - public List buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping) + public void doBuildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping, Consumer recordCustomizer) { if(!StringUtils.hasContent(csv)) { throw (new IllegalArgumentException("Empty csv value was provided.")); } - List rs = new ArrayList<>(); try { /////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -82,7 +118,7 @@ public class CsvToQRecordAdapter // put values from the CSV record into a map of header -> value // ////////////////////////////////////////////////////////////////// Map csvValues = new HashMap<>(); - for(int i=0; i recordCustomizer, QRecord qRecord) + { + if(recordCustomizer != null) + { + recordCustomizer.accept(qRecord); + } } @@ -165,7 +216,7 @@ public class CsvToQRecordAdapter for(String header : headers) { - String headerToUse = header; + String headerToUse = header; String headerWithoutSuffix = header.replaceFirst(" \\d+$", ""); if(countsByHeader.containsKey(headerWithoutSuffix)) @@ -183,4 +234,22 @@ public class CsvToQRecordAdapter return (rs); } + + + /******************************************************************************* + ** Add a record - either to the pipe, or list, whichever we're building. + *******************************************************************************/ + private void addRecord(QRecord record) + { + if(recordPipe != null) + { + recordPipe.addRecord(record); + } + + if(recordList != null) + { + recordList.add(record); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java index a6460bff..80e5d531 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java @@ -35,7 +35,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; ** Output data container for the RunBackendStep action ** *******************************************************************************/ -public class RunBackendStepOutput extends AbstractActionOutput +public class RunBackendStepOutput extends AbstractActionOutput implements Serializable { private ProcessState processState; private Exception exception; // todo - make optional diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java index d2ec4f15..24320d98 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.insert; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; *******************************************************************************/ public class InsertInput extends AbstractTableActionInput { + private QBackendTransaction transaction; private List records; @@ -57,6 +59,39 @@ public class InsertInput extends AbstractTableActionInput + /******************************************************************************* + ** Getter for transaction + ** + *******************************************************************************/ + public QBackendTransaction getTransaction() + { + return transaction; + } + + + + /******************************************************************************* + ** Setter for transaction + ** + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + + + /******************************************************************************* + ** Fluent setter for transaction + ** + *******************************************************************************/ + public InsertInput withTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + return (this); + } + + + /******************************************************************************* ** Getter for records ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java index 9e65ade9..6f8ce386 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java @@ -23,10 +23,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.util.List; -import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.utils.SleepUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -58,35 +56,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface @Override public void addRecord(QRecord record) { - if(!recordPipe.addRecord(record)) - { - do - { - LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); - SleepUtils.sleep(10, TimeUnit.MILLISECONDS); - } - while(!recordPipe.addRecord(record)); - LOG.debug("Done blocking."); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void blockIfPipeIsTooFull() - { - if(recordPipe.countAvailableRecords() >= 100_000) - { - LOG.info("Record pipe is kinda full. Blocking for a bit"); - do - { - SleepUtils.sleep(10, TimeUnit.MILLISECONDS); - } - while(recordPipe.countAvailableRecords() >= 10_000); - LOG.info("Done blocking."); - } + recordPipe.addRecord(record); } @@ -98,7 +68,6 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface public void addRecords(List records) { recordPipe.addRecords(records); - blockIfPipeIsTooFull(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java index 79bc0d73..435a13ce 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java @@ -57,20 +57,25 @@ import org.json.JSONObject; *******************************************************************************/ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface { - private static final Logger logger = LogManager.getLogger(Auth0AuthenticationModule.class); + private static final Logger LOG = LogManager.getLogger(Auth0AuthenticationModule.class); - private static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 300; + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // 30 minutes - ideally this would be lower, but right now we've been dealing with re-validation issues... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + public static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 1800; public static final String AUTH0_ID_TOKEN_KEY = "sessionId"; public static final String TOKEN_NOT_PROVIDED_ERROR = "Id Token was not provided"; - public static final String COULD_NOT_DECODE_ERROR = "Unable to decode id token"; - public static final String EXPIRED_TOKEN_ERROR = "Token has expired"; - public static final String INVALID_TOKEN_ERROR = "An invalid token was provided"; + public static final String COULD_NOT_DECODE_ERROR = "Unable to decode id token"; + public static final String EXPIRED_TOKEN_ERROR = "Token has expired"; + public static final String INVALID_TOKEN_ERROR = "An invalid token was provided"; private Instant now; + + /******************************************************************************* ** *******************************************************************************/ @@ -83,7 +88,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface String idToken = context.get(AUTH0_ID_TOKEN_KEY); if(idToken == null) { - logger.warn(TOKEN_NOT_PROVIDED_ERROR); + LOG.warn(TOKEN_NOT_PROVIDED_ERROR); throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR)); } @@ -97,7 +102,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // then call method to check more session validity // ///////////////////////////////////////////////////// QSession qSession = buildQSessionFromToken(idToken); - if(isSessionValid(qSession)) + if(isSessionValid(qInstance, qSession)) { return (qSession); } @@ -112,7 +117,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // put now into state so we dont check until next interval passes // /////////////////////////////////////////////////////////////////// StateProviderInterface spi = getStateProvider(); - Auth0StateKey key = new Auth0StateKey(qSession.getIdReference()); + Auth0StateKey key = new Auth0StateKey(qSession.getIdReference()); spi.put(key, Instant.now()); return (qSession); @@ -122,12 +127,12 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface //////////////////////////////// // could not decode the token // //////////////////////////////// - logger.warn(COULD_NOT_DECODE_ERROR, jde); + LOG.warn(COULD_NOT_DECODE_ERROR, jde); throw (new QAuthenticationException(COULD_NOT_DECODE_ERROR)); } catch(TokenExpiredException tee) { - logger.info(EXPIRED_TOKEN_ERROR, tee); + LOG.info(EXPIRED_TOKEN_ERROR, tee); throw (new QAuthenticationException(EXPIRED_TOKEN_ERROR)); } catch(JWTVerificationException | JwkException jve) @@ -135,7 +140,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface /////////////////////////////////////////// // token had invalid signature or claims // /////////////////////////////////////////// - logger.warn(INVALID_TOKEN_ERROR, jve); + LOG.warn(INVALID_TOKEN_ERROR, jve); throw (new QAuthenticationException(INVALID_TOKEN_ERROR)); } catch(Exception e) @@ -144,7 +149,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // ¯\_(ツ)_/¯ // //////////////// String message = "An unknown error occurred"; - logger.error(message, e); + LOG.error(message, e); throw (new QAuthenticationException(message)); } } @@ -155,16 +160,16 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface ** *******************************************************************************/ @Override - public boolean isSessionValid(QSession session) + public boolean isSessionValid(QInstance instance, QSession session) { if(session == null) { return (false); } - StateProviderInterface spi = getStateProvider(); - Auth0StateKey key = new Auth0StateKey(session.getIdReference()); - Optional lastTimeCheckedOptional = spi.get(Instant.class, key); + StateProviderInterface spi = getStateProvider(); + Auth0StateKey key = new Auth0StateKey(session.getIdReference()); + Optional lastTimeCheckedOptional = spi.get(Instant.class, key); if(lastTimeCheckedOptional.isPresent()) { Instant lastTimeChecked = lastTimeCheckedOptional.get(); @@ -174,7 +179,28 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // - so this is basically saying, if the time between the last time we checked the token and // // right now is more than ID_TOKEN_VALIDATION_INTERVAL_SECTIONS, then session needs revalidated // /////////////////////////////////////////////////////////////////////////////////////////////////// - return (Duration.between(lastTimeChecked, Instant.now()).compareTo(Duration.ofSeconds(ID_TOKEN_VALIDATION_INTERVAL_SECONDS)) < 0); + if(Duration.between(lastTimeChecked, Instant.now()).compareTo(Duration.ofSeconds(ID_TOKEN_VALIDATION_INTERVAL_SECONDS)) < 0) + { + return (true); + } + + try + { + LOG.debug("Re-validating token due to validation interval being passed: " + session.getIdReference()); + revalidateToken(instance, session.getIdReference()); + + ////////////////////////////////////////////////////////////////// + // update the timestamp in state provider, to avoid re-checking // + ////////////////////////////////////////////////////////////////// + spi.put(key, Instant.now()); + + return (true); + } + catch(Exception e) + { + LOG.warn(INVALID_TOKEN_ERROR, e); + return (false); + } } return (false); @@ -190,10 +216,10 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface { Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication(); - DecodedJWT jwt = JWT.decode(idToken); - JwkProvider provider = new UrlJwkProvider(metaData.getBaseUrl()); - Jwk jwk = provider.get(jwt.getKeyId()); - Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null); + DecodedJWT jwt = JWT.decode(idToken); + JwkProvider provider = new UrlJwkProvider(metaData.getBaseUrl()); + Jwk jwk = provider.get(jwt.getKeyId()); + Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null); JWTVerifier verifier = JWT.require(algorithm) .withIssuer(metaData.getBaseUrl()) .build(); @@ -217,20 +243,31 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface //////////////////////////////////// // decode and extract the payload // //////////////////////////////////// - DecodedJWT jwt = JWT.decode(idToken); - Base64.Decoder decoder = Base64.getUrlDecoder(); - String payloadString = new String(decoder.decode(jwt.getPayload())); - JSONObject payload = new JSONObject(payloadString); + DecodedJWT jwt = JWT.decode(idToken); + Base64.Decoder decoder = Base64.getUrlDecoder(); + String payloadString = new String(decoder.decode(jwt.getPayload())); + JSONObject payload = new JSONObject(payloadString); QUser qUser = new QUser(); - qUser.setFullName(payload.getString("name")); + if(payload.has("name")) + { + qUser.setFullName(payload.getString("name")); + } + else + { + qUser.setFullName("Unknown"); + } + if(payload.has("email")) { qUser.setIdReference(payload.getString("email")); } else { - qUser.setIdReference(payload.getString("nickname")); + if(payload.has("sub")) + { + qUser.setIdReference(payload.getString("sub")); + } } QSession qSession = new QSession(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java index 442941b5..8a0e7078 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java @@ -66,7 +66,7 @@ public class FullyAnonymousAuthenticationModule implements QAuthenticationModule ** *******************************************************************************/ @Override - public boolean isSessionValid(QSession session) + public boolean isSessionValid(QInstance instance, QSession session) { return session != null; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java index 21b25c2a..f490d0a4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java @@ -63,7 +63,7 @@ public class MockAuthenticationModule implements QAuthenticationModuleInterface ** *******************************************************************************/ @Override - public boolean isSessionValid(QSession session) + public boolean isSessionValid(QInstance instance, QSession session) { if(session == null) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java index ec5db589..75c1535f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java @@ -43,5 +43,5 @@ public interface QAuthenticationModuleInterface /******************************************************************************* ** *******************************************************************************/ - boolean isSessionValid(QSession session); + boolean isSessionValid(QInstance instance, QSession session); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java index 20c27808..9b0d0687 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -40,6 +41,8 @@ public class BasicETLExtractFunction implements BackendStep { private static final Logger LOG = LogManager.getLogger(BasicETLExtractFunction.class); + private RecordPipe recordPipe = null; + /******************************************************************************* @@ -64,10 +67,35 @@ public class BasicETLExtractFunction implements BackendStep // queryRequest.setFilter(JsonUtils.toObject(filter, QQueryFilter.class)); // } + ////////////////////////////////////////////////////////////////////// + // if the caller gave us a record pipe, pass it to the query action // + ////////////////////////////////////////////////////////////////////// + if (recordPipe != null) + { + queryInput.setRecordPipe(recordPipe); + } + QueryAction queryAction = new QueryAction(); QueryOutput queryOutput = queryAction.execute(queryInput); - runBackendStepOutput.setRecords(queryOutput.getRecords()); - LOG.info("Query on table " + tableName + " produced " + queryOutput.getRecords().size() + " records."); + if (recordPipe == null) + { + //////////////////////////////////////////////////////////////////////////// + // only return the records (and log about them) if there's no record pipe // + //////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.setRecords(queryOutput.getRecords()); + LOG.info("Query on table " + tableName + " produced " + queryOutput.getRecords().size() + " records."); + } + } + + + + /******************************************************************************* + ** Setter for recordPipe + ** + *******************************************************************************/ + public void setRecordPipe(RecordPipe recordPipe) + { + this.recordPipe = recordPipe; } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java index 997e377b..0ce1c572 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -44,6 +45,8 @@ public class BasicETLLoadFunction implements BackendStep { private static final Logger LOG = LogManager.getLogger(BasicETLLoadFunction.class); + private QBackendTransaction transaction; + /******************************************************************************* @@ -86,10 +89,11 @@ public class BasicETLLoadFunction implements BackendStep insertInput.setSession(runBackendStepInput.getSession()); insertInput.setTableName(table); insertInput.setRecords(page); + insertInput.setTransaction(transaction); InsertAction insertAction = new InsertAction(); InsertOutput insertOutput = insertAction.execute(insertInput); - outputRecords.addAll(insertOutput.getRecords()); + // todo - this is to avoid garbage leak in state provider... outputRecords.addAll(insertOutput.getRecords()); recordsInserted += insertOutput.getRecords().size(); } @@ -97,4 +101,15 @@ public class BasicETLLoadFunction implements BackendStep runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsInserted); } + + + /******************************************************************************* + ** Setter for transaction + ** + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java new file mode 100644 index 00000000..bb4337dc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java @@ -0,0 +1,240 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed; + + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLExtractFunction; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLLoadFunction; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLTransformFunction; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Backend step to do a streamed ETL + *******************************************************************************/ +public class StreamedETLBackendStep implements BackendStep +{ + private static final Logger LOG = LogManager.getLogger(StreamedETLBackendStep.class); + + private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000; + + private static final int MAX_SLEEP_MS = 1000; + private static final int INIT_SLEEP_MS = 10; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + QBackendTransaction transaction = openTransaction(runBackendStepInput); + + try + { + RecordPipe recordPipe = new RecordPipe(); + BasicETLExtractFunction basicETLExtractFunction = new BasicETLExtractFunction(); + basicETLExtractFunction.setRecordPipe(recordPipe); + + ////////////////////////////////////////// + // run the query action as an async job // + ////////////////////////////////////////// + AsyncJobManager asyncJobManager = new AsyncJobManager(); + String queryJobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> + { + basicETLExtractFunction.run(runBackendStepInput, runBackendStepOutput); + return (runBackendStepOutput); + }); + LOG.info("Started query job [" + queryJobUUID + "] for report"); + + AsyncJobState queryJobState = AsyncJobState.RUNNING; + AsyncJobStatus asyncJobStatus = null; + + long recordCount = 0; + int nextSleepMillis = INIT_SLEEP_MS; + long lastReceivedRecordsAt = System.currentTimeMillis(); + long jobStartTime = System.currentTimeMillis(); + + while(queryJobState.equals(AsyncJobState.RUNNING)) + { + if(recordPipe.countAvailableRecords() == 0) + { + /////////////////////////////////////////////////////////// + // if the pipe is empty, sleep to let the producer work. // + // todo - smarter sleep? like get notified vs. sleep? // + /////////////////////////////////////////////////////////// + LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); + SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); + nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS); + + long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt; + if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS) + { + throw (new QException("Query action appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago).")); + } + } + else + { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + lastReceivedRecordsAt = System.currentTimeMillis(); + nextSleepMillis = INIT_SLEEP_MS; + + recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction); + + LOG.info(String.format("Processed %,d records so far", recordCount)); + } + + //////////////////////////////////// + // refresh the query job's status // + //////////////////////////////////// + Optional optionalAsyncJobStatus = asyncJobManager.getJobStatus(queryJobUUID); + if(optionalAsyncJobStatus.isEmpty()) + { + ///////////////////////////////////////////////// + // todo - ... maybe some version of try-again? // + ///////////////////////////////////////////////// + throw (new QException("Could not get status of report query job [" + queryJobUUID + "]")); + } + asyncJobStatus = optionalAsyncJobStatus.get(); + queryJobState = asyncJobStatus.getState(); + } + + LOG.info("Query job [" + queryJobUUID + "] for ETL completed with status: " + asyncJobStatus); + + ////////////////////////////////////////////////////// + // send the final records to transform & load steps // + ////////////////////////////////////////////////////// + recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction); + + ///////////////////// + // commit the work // + ///////////////////// + transaction.commit(); + + long reportEndTime = System.currentTimeMillis(); + LOG.info(String.format("Processed %,d records", recordCount) + + String.format(" at end of ETL job in %,d ms (%.2f records/second).", (reportEndTime - jobStartTime), 1000d * (recordCount / (.001d + (reportEndTime - jobStartTime))))); + + runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + } + catch(Exception e) + { + //////////////////////////////////////////////////////////////////////////////// + // rollback the work, then re-throw the error for up-stream to catch & report // + //////////////////////////////////////////////////////////////////////////////// + transaction.rollback(); + throw (e); + } + finally + { + //////////////////////////////////////////////////////////// + // always close our transactions (e.g., jdbc connections) // + //////////////////////////////////////////////////////////// + transaction.close(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QBackendTransaction openTransaction(RunBackendStepInput runBackendStepInput) throws QException + { + InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); + + insertInput.setSession(runBackendStepInput.getSession()); + insertInput.setTableName(runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE)); + + return new InsertAction().openTransaction(insertInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private int consumeRecordsFromPipe(RecordPipe recordPipe, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, QBackendTransaction transaction) throws QException + { + List qRecords = recordPipe.consumeAvailableRecords(); + + preTransform(qRecords, runBackendStepInput, runBackendStepOutput); + + runBackendStepInput.setRecords(qRecords); + new BasicETLTransformFunction().run(runBackendStepInput, runBackendStepOutput); + + postTransform(qRecords, runBackendStepInput, runBackendStepOutput); + + runBackendStepInput.setRecords(runBackendStepOutput.getRecords()); + BasicETLLoadFunction basicETLLoadFunction = new BasicETLLoadFunction(); + basicETLLoadFunction.setTransaction(transaction); + basicETLLoadFunction.run(runBackendStepInput, runBackendStepOutput); + + return (qRecords.size()); + } + + + + /******************************************************************************* + ** Customization point for subclasses of this step. + *******************************************************************************/ + protected void preTransform(List qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + + /******************************************************************************* + ** Customization point for subclasses of this step. + *******************************************************************************/ + protected void postTransform(List qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + //////////////////////// + // noop in base class // + //////////////////////// + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java new file mode 100644 index 00000000..2fa636e7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed; + + +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; + + +/******************************************************************************* + ** Definition for Streamed ETL process. + *******************************************************************************/ +public class StreamedETLProcess +{ + public static final String PROCESS_NAME = "etl.streamed"; + + public static final String FUNCTION_NAME_ETL = "streamedETL"; + + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_MAPPING_JSON = "mappingJSON"; + public static final String FIELD_RECORD_COUNT = "recordCount"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QProcessMetaData defineProcessMetaData() + { + QStepMetaData etlFunction = new QBackendStepMetaData() + .withName(FUNCTION_NAME_ETL) + .withCode(new QCodeReference() + .withName(StreamedETLBackendStep.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.BACKEND_STEP)) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)) + .withField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING)) + .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING))) + .withOutputMetaData(new QFunctionOutputMetaData() + .addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); + + return new QProcessMetaData() + .withName(PROCESS_NAME) + .addStep(etlFunction); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java index f29a62fa..71d2807f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java @@ -80,7 +80,7 @@ class ReportActionTest public void testBigger() throws Exception { // int recordCount = 2_000_000; // to really stress locally, use this. - int recordCount = 200_000; + int recordCount = 50_000; String filename = "/tmp/ReportActionTest.csv"; runReport(recordCount, filename, ReportFormat.CSV, false); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java index 99cbb6b6..049d3c57 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.modules.authentication; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException; @@ -31,7 +32,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.Auth0AuthenticationMetaData; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider; -import com.kingsrook.qqq.backend.core.state.StateProviderInterface; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.AUTH0_ID_TOKEN_KEY; @@ -40,7 +40,8 @@ import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0Authent import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.INVALID_TOKEN_ERROR; import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.TOKEN_NOT_PROVIDED_ERROR; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -49,9 +50,9 @@ import static org.junit.jupiter.api.Assertions.fail; *******************************************************************************/ public class Auth0AuthenticationModuleTest { - private static final String VALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA"; - private static final String INVALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA-thismakesinvalid"; - private static final String EXPIRED_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE4VDIxOjM4OjE1LjM4NloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MTgwNDc3LCJleHAiOjE2NTgyMTY0NzcsIm5vbmNlIjoiVkZkQlYzWmplR2hvY1cwMk9WZEtabHBLU0c1K1ZXbElhMEV3VkZaeFpVdEJVMDErZUZaT1RtMTNiZz09In0.fU7EwUgNrupOPz_PX_aQKON2xG1-LWD85xVo1Bn41WNEek-iMyJoch8l6NUihi7Bou14BoOfeWIG_sMqsLHqI2Pk7el7l1kigsjURx0wpiXadBt8piMxdIlxdToZEMuZCBzg7eJvXh4sM8tlV5cm0gPa6FT9Ih3VGJajNlXi5BcYS_JRpIvFvHn8-Bxj4KiAlZ5XPPkopjnDgP8kFfc4cMn_nxDkqWYlhj-5TaGW2xCLC9Qr_9UNxX0fm-CkKjYs3Z5ezbiXNkc-bxrCYvxeBeDPf8-T3EqrxCRVqCZSJ85BHdOc_E7UZC_g8bNj0umoplGwlCbzO4XIuOO-KlIaOg"; + private static final String VALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA"; + private static final String INVALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA-thismakesinvalid"; + private static final String EXPIRED_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE4VDIxOjM4OjE1LjM4NloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MTgwNDc3LCJleHAiOjE2NTgyMTY0NzcsIm5vbmNlIjoiVkZkQlYzWmplR2hvY1cwMk9WZEtabHBLU0c1K1ZXbElhMEV3VkZaeFpVdEJVMDErZUZaT1RtMTNiZz09In0.fU7EwUgNrupOPz_PX_aQKON2xG1-LWD85xVo1Bn41WNEek-iMyJoch8l6NUihi7Bou14BoOfeWIG_sMqsLHqI2Pk7el7l1kigsjURx0wpiXadBt8piMxdIlxdToZEMuZCBzg7eJvXh4sM8tlV5cm0gPa6FT9Ih3VGJajNlXi5BcYS_JRpIvFvHn8-Bxj4KiAlZ5XPPkopjnDgP8kFfc4cMn_nxDkqWYlhj-5TaGW2xCLC9Qr_9UNxX0fm-CkKjYs3Z5ezbiXNkc-bxrCYvxeBeDPf8-T3EqrxCRVqCZSJ85BHdOc_E7UZC_g8bNj0umoplGwlCbzO4XIuOO-KlIaOg"; private static final String UNDECODABLE_TOKEN = "UNDECODABLE"; public static final String AUTH0_BASE_URL = "https://kingsrook.us.auth0.com/"; @@ -59,32 +60,79 @@ public class Auth0AuthenticationModuleTest /******************************************************************************* - ** Test a valid token where 'now' is set to a time that would be valid for it + ** Test an expired token where 'now' is set to a time that would not require it to be + ** re-checked, so it'll show as valid ** *******************************************************************************/ @Test - public void testLastTimeChecked() throws QAuthenticationException + public void testLastTimeCheckedNow() { - ////////////////////////////////////////////////////////// - // Tuesday, July 19, 2022 12:40:27.299 PM GMT-05:00 DST // - ////////////////////////////////////////////////////////// - Instant now = Instant.now(); + assertTrue(testLastTimeChecked(Instant.now(), UNDECODABLE_TOKEN), "A session just checked 'now' should always be valid"); + } - ///////////////////////////////////////////////////////// - // put the 'now' from the past into the state provider // - ///////////////////////////////////////////////////////// - StateProviderInterface spi = InMemoryStateProvider.getInstance(); - Auth0AuthenticationModule.Auth0StateKey key = new Auth0AuthenticationModule.Auth0StateKey(VALID_TOKEN); - spi.put(key, now); + + + /******************************************************************************* + ** Test an expired token where 'now' is set to a time that would not require it to be + ** re-checked, so it'll show as valid + ** + *******************************************************************************/ + @Test + public void testLastTimeCheckedJustUnderThreshold() + { + Instant underThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS - 60, ChronoUnit.SECONDS); + assertTrue(testLastTimeChecked(underThreshold, INVALID_TOKEN), "A session checked under threshold should be valid"); + } + + + + /******************************************************************************* + ** Test an expired token where 'now' is set to a time that would require it to be + ** re-checked + ** + *******************************************************************************/ + @Test + public void testLastTimeCheckedJustOverThreshold() + { + Instant overThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS + 60, ChronoUnit.SECONDS); + assertFalse(testLastTimeChecked(overThreshold, INVALID_TOKEN), "A session checked over threshold should be re-validated, and in this case, not be valid."); + } + + + + /******************************************************************************* + ** Test an expired token where 'now' is set to a time that would require it to be + ** re-checked + ** + *******************************************************************************/ + @Test + public void testLastTimeCheckedOverThresholdAndUndecodable() + { + Instant overThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS + 60, ChronoUnit.SECONDS); + assertFalse(testLastTimeChecked(overThreshold, UNDECODABLE_TOKEN), "A session checked over threshold should be re-validated, and in this case, not be valid."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testLastTimeChecked(Instant lastTimeChecked, String token) + { + ///////////////////////////////////////////////////////////// + // put the input last-time-checked into the state provider // + ///////////////////////////////////////////////////////////// + Auth0AuthenticationModule.Auth0StateKey key = new Auth0AuthenticationModule.Auth0StateKey(token); + InMemoryStateProvider.getInstance().put(key, lastTimeChecked); ////////////////////// // build up session // ////////////////////// QSession session = new QSession(); - session.setIdReference(VALID_TOKEN); + session.setIdReference(token); Auth0AuthenticationModule auth0AuthenticationModule = new Auth0AuthenticationModule(); - assertEquals(true, auth0AuthenticationModule.isSessionValid(session), "Session should return as still valid."); + return (auth0AuthenticationModule.isSessionValid(getQInstance(), session)); } @@ -114,7 +162,7 @@ public class Auth0AuthenticationModuleTest /******************************************************************************* - ** Test failure case, token cant be decoded + ** Test failure case, token can't be decoded ** *******************************************************************************/ @Test @@ -220,4 +268,5 @@ public class Auth0AuthenticationModuleTest qInstance.setAuthentication(authenticationMetaData); return (qInstance); } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java index 68c1e7d8..f775511f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java @@ -49,8 +49,8 @@ public class FullyAnonymousAuthenticationModuleTest assertNotNull(session.getIdReference(), "Session id ref should not be null"); assertNotNull(session.getUser(), "Session User should not be null"); assertNotNull(session.getUser().getIdReference(), "Session User id ref should not be null"); - assertTrue(fullyAnonymousAuthenticationModule.isSessionValid(session), "Any session should be valid"); - assertFalse(fullyAnonymousAuthenticationModule.isSessionValid(null), "null should be not valid"); + assertTrue(fullyAnonymousAuthenticationModule.isSessionValid(null, session), "Any session should be valid"); + assertFalse(fullyAnonymousAuthenticationModule.isSessionValid(null, null), "null should be not valid"); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java new file mode 100644 index 00000000..97d756c9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed; + + +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for BasicETLProcess + *******************************************************************************/ +class StreamedETLProcessTest +{ + + /******************************************************************************* + ** Simplest happy path + *******************************************************************************/ + @Test + public void test() throws QException + { + RunProcessInput request = new RunProcessInput(TestUtils.defineInstance()); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(StreamedETLProcess.PROCESS_NAME); + request.addValue(StreamedETLProcess.FIELD_SOURCE_TABLE, TestUtils.defineTablePerson().getName()); + request.addValue(StreamedETLProcess.FIELD_DESTINATION_TABLE, TestUtils.definePersonFileTable().getName()); + request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, ""); + + RunProcessOutput result = new RunProcessAction().execute(request); + assertNotNull(result); + assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process"); + assertTrue(result.getException().isEmpty()); + } + + + + /******************************************************************************* + ** Basic example of doing a mapping transformation + *******************************************************************************/ + @Test + public void testMappingTransformation() throws QException + { + RunProcessInput request = new RunProcessInput(TestUtils.defineInstance()); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(StreamedETLProcess.PROCESS_NAME); + request.addValue(StreamedETLProcess.FIELD_SOURCE_TABLE, TestUtils.definePersonFileTable().getName()); + request.addValue(StreamedETLProcess.FIELD_DESTINATION_TABLE, TestUtils.defineTableIdAndNameOnly().getName()); + + /////////////////////////////////////////////////////////////////////////////////////// + // define our mapping from destination-table field names to source-table field names // + /////////////////////////////////////////////////////////////////////////////////////// + QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping().withMapping("name", "firstName"); + // request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping.getMapping())); + request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping)); + + RunProcessOutput result = new RunProcessAction().execute(request); + assertNotNull(result); + assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process"); + assertTrue(result.getException().isEmpty()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index d7f6b5d0..3fc3e6b0 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthentic import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; /******************************************************************************* @@ -91,6 +92,7 @@ public class TestUtils qInstance.addProcess(defineProcessGreetPeopleInteractive()); qInstance.addProcess(defineProcessAddToPeoplesAge()); qInstance.addProcess(new BasicETLProcess().defineProcessMetaData()); + qInstance.addProcess(new StreamedETLProcess().defineProcessMetaData()); System.out.println(new QInstanceAdapter().qInstanceToJson(qInstance)); diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 84341c17..02b26622 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -201,10 +201,16 @@ public abstract class AbstractBaseFilesystemAction String fileContents = IOUtils.toString(readFile(file)); fileContents = customizeFileContentsAfterReading(table, fileContents); - List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - - queryOutput.addRecords(recordsInFile); + if(queryInput.getRecordPipe() != null) + { + new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> addBackendDetailsToRecord(record, file))); + } + else + { + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + queryOutput.addRecords(recordsInFile); + } break; } case JSON: @@ -212,6 +218,7 @@ public abstract class AbstractBaseFilesystemAction String fileContents = IOUtils.toString(readFile(file)); fileContents = customizeFileContentsAfterReading(table, fileContents); + // todo - pipe support!! List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); addBackendDetailsToRecords(recordsInFile, file); @@ -241,10 +248,17 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ protected void addBackendDetailsToRecords(List recordsInFile, FILE file) { - recordsInFile.forEach(record -> - { - record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file)); - }); + recordsInFile.forEach(r -> addBackendDetailsToRecord(r, file)); + } + + + + /******************************************************************************* + ** Add backend details to a record about the file that it is in. + *******************************************************************************/ + protected void addBackendDetailsToRecord(QRecord record, FILE file) + { + record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file)); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java index f455f018..7add637a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java @@ -94,11 +94,13 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep String moveOrDelete = runBackendStepInput.getValueString(FIELD_MOVE_OR_DELETE); if(VALUE_DELETE.equals(moveOrDelete)) { + LOG.info("Deleting ETL source file: " + sourceFile); actionBase.deleteFile(runBackendStepInput.getInstance(), table, sourceFile); } else if(VALUE_MOVE.equals(moveOrDelete)) { String destinationForMoves = runBackendStepInput.getValueString(FIELD_DESTINATION_FOR_MOVES); + LOG.info("Moving ETL source file: " + sourceFile + " to " + destinationForMoves); if(!StringUtils.hasContent(destinationForMoves)) { throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value.")); diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStep.java new file mode 100644 index 00000000..8a255d88 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStep.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed; + + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLBackendStep; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic.BasicETLCollectSourceFileNamesStep; + + +/******************************************************************************* + ** Extension to the base StreamedETLBackendStep, unique for where the source + ** table is a filesystem, where we want/need to collect the filenames that were + ** processed in the Extract step, so they can be passed into the cleanup step. + ** + ** Similar in purpose to the BasicETLCollectSourceFileNamesStep - only in this + ** case, due to the streaming behavior of the StreamedETLProcess, we can't really + ** inject this code as a separate backend step - so instead we extend that step, + ** and override its postTransform method to intercept the records & file names. + *******************************************************************************/ +public class StreamedETLFilesystemBackendStep extends StreamedETLBackendStep +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected void preTransform(List qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + Set sourceFiles = qRecords.stream() + .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) + .collect(Collectors.toSet()); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // expect that we'll be called on multiple "pages" of records as they run through the pipe. // + // each time we're called, we need to: // + // - get the unique file paths in this list of records // + // - if we previously set the list of file names in the output, then split that value up and add those names to the set we see now // + // - set the list of name (joined by commas) in the output // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + String existingListOfFileNames = runBackendStepOutput.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS); + if(existingListOfFileNames != null) + { + sourceFiles.addAll(List.of(existingListOfFileNames.split(","))); + } + runBackendStepOutput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java index b791d355..9b537f95 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -137,7 +137,7 @@ public class S3Utils //////////////////////////////////////////////////////////////////////////////// if(key.endsWith("/")) { - LOG.debug("Skipping file [{}] because it is a folder", key); + // LOG.debug("Skipping file [{}] because it is a folder", key); continue; } @@ -146,7 +146,7 @@ public class S3Utils /////////////////////////////////////////// if(!pathMatcher.matches(Path.of(URI.create("file:///" + key)))) { - LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob); + // LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob); continue; } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index fd7d0714..103df995 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -24,20 +24,27 @@ package com.kingsrook.qqq.backend.module.filesystem; import java.io.File; import java.io.IOException; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed.StreamedETLFilesystemBackendStep; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; @@ -49,10 +56,16 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; - public static final String BACKEND_NAME_S3 = "s3"; - public static final String TABLE_NAME_PERSON_LOCAL_FS = "person"; - public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String BACKEND_NAME_MOCK = "mock"; + + public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; + public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; + public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; + + public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; /////////////////////////////////////////////////////////////////// // shouldn't be accessed directly, as we append a counter to it. // @@ -112,14 +125,18 @@ public class TestUtils /******************************************************************************* ** *******************************************************************************/ - public static QInstance defineInstance() throws QInstanceValidationException + public static QInstance defineInstance() throws QException { QInstance qInstance = new QInstance(); qInstance.setAuthentication(defineAuthentication()); qInstance.addBackend(defineLocalFilesystemBackend()); qInstance.addTable(defineLocalFilesystemJSONPersonTable()); + qInstance.addTable(defineLocalFilesystemCSVPersonTable()); qInstance.addBackend(defineS3Backend()); qInstance.addTable(defineS3CSVPersonTable()); + qInstance.addBackend(defineMockBackend()); + qInstance.addTable(defineMockPersonTable()); + qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); new QInstanceValidator().validate(qInstance); @@ -159,21 +176,55 @@ public class TestUtils public static QTableMetaData defineLocalFilesystemJSONPersonTable() { return new QTableMetaData() - .withName(TABLE_NAME_PERSON_LOCAL_FS) + .withName(TABLE_NAME_PERSON_LOCAL_FS_JSON) .withLabel("Person") .withBackendName(defineLocalFilesystemBackend().getName()) .withPrimaryKeyField("id") - .withField(new QFieldMetaData("id", QFieldType.INTEGER)) - .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) - .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) - .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) - .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) - .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) - .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withFields(defineCommonPersonTableFields()) .withBackendDetails(new FilesystemTableBackendDetails() .withBasePath("persons") .withRecordFormat(RecordFormat.JSON) .withCardinality(Cardinality.MANY) + .withGlob("*.json") + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List defineCommonPersonTableFields() + { + return (List.of( + new QFieldMetaData("id", QFieldType.INTEGER), + new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"), + new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"), + new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"), + new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"), + new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"), + new QFieldMetaData("email", QFieldType.STRING) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineLocalFilesystemCSVPersonTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON_LOCAL_FS_CSV) + .withLabel("Person") + .withBackendName(defineLocalFilesystemBackend().getName()) + .withPrimaryKeyField("id") + .withFields(defineCommonPersonTableFields()) + .withBackendDetails(new FilesystemTableBackendDetails() + .withBasePath("persons-csv") + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) + .withGlob("*.csv") ); } @@ -202,13 +253,7 @@ public class TestUtils .withLabel("Person S3 Table") .withBackendName(defineS3Backend().getName()) .withPrimaryKeyField("id") - .withField(new QFieldMetaData("id", QFieldType.INTEGER)) - .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) - .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) - .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) - .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) - .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) - .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withFields(defineCommonPersonTableFields()) .withBackendDetails(new S3TableBackendDetails() .withRecordFormat(RecordFormat.CSV) .withCardinality(Cardinality.MANY) @@ -220,7 +265,52 @@ public class TestUtils /******************************************************************************* ** *******************************************************************************/ - public static QSession getMockSession() throws QInstanceValidationException + public static QBackendMetaData defineMockBackend() + { + return (new QBackendMetaData() + .withBackendType("mock") + .withName(BACKEND_NAME_MOCK)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineMockPersonTable() + { + return (new QTableMetaData() + .withName(TABLE_NAME_PERSON_MOCK) + .withLabel("Person Mock Table") + .withBackendName(BACKEND_NAME_MOCK) + .withPrimaryKeyField("id") + .withFields(defineCommonPersonTableFields())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QProcessMetaData defineStreamedLocalCsvToMockETLProcess() throws QException + { + QProcessMetaData qProcessMetaData = new StreamedETLProcess().defineProcessMetaData(); + qProcessMetaData.setName(PROCESS_NAME_STREAMED_ETL); + QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL); + backendStep.setCode(new QCodeReference(StreamedETLFilesystemBackendStep.class)); + + backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_SOURCE_TABLE).setDefaultValue(TABLE_NAME_PERSON_LOCAL_FS_CSV); + backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_DESTINATION_TABLE).setDefaultValue(TABLE_NAME_PERSON_MOCK); + + return (qProcessMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QSession getMockSession() throws QException { MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); return (mockAuthenticationModule.createSession(defineInstance(), null)); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index 17318fd3..ea52bc97 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -70,7 +70,7 @@ public class FilesystemBackendModuleTest public void testDeleteFile() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); ///////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then delete one, then re-list, and assert that we have one fewer // @@ -94,7 +94,7 @@ public class FilesystemBackendModuleTest public void testDeleteFileDoesNotExist() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // @@ -120,7 +120,7 @@ public class FilesystemBackendModuleTest public void testMoveFile() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); String subPath = basePath + File.separator + "subdir"; @@ -157,7 +157,7 @@ public class FilesystemBackendModuleTest public void testMoveFileDoesNotExit() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); String subPath = basePath + File.separator + "subdir"; List filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 755ab477..94cbfbf7 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -80,7 +80,8 @@ public class FilesystemActionTest fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module"); } - writePersonFiles(baseDirectory); + writePersonJSONFiles(baseDirectory); + writePersonCSVFiles(baseDirectory); } @@ -88,7 +89,7 @@ public class FilesystemActionTest /******************************************************************************* ** Write some data files into the directory for the filesystem module. *******************************************************************************/ - private void writePersonFiles(File baseDirectory) throws IOException + private void writePersonJSONFiles(File baseDirectory) throws IOException { String fullPath = baseDirectory.getAbsolutePath(); if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) @@ -118,6 +119,38 @@ public class FilesystemActionTest + /******************************************************************************* + ** Write some data files into the directory for the filesystem module. + *******************************************************************************/ + private void writePersonCSVFiles(File baseDirectory) throws IOException + { + String fullPath = baseDirectory.getAbsolutePath(); + if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) + { + if (StringUtils.hasContent(details.getBasePath())) + { + fullPath += File.separatorChar + details.getBasePath(); + } + } + fullPath += File.separatorChar; + + String csvData1 = """ + "id","createDate","modifyDate","firstName","lastName","birthDate","email" + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com" + "2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com" + """; + FileUtils.writeStringToFile(new File(fullPath + "FILE-1.csv"), csvData1); + + String csvData2 = """ + "id","createDate","modifyDate","firstName","lastName","birthDate","email" + "3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com" + "4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com" + "5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com\""""; // intentionally no \n at EOL here + FileUtils.writeStringToFile(new File(fullPath + "FILE-2.csv"), csvData2); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index cc90a4bc..90771e43 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -71,7 +71,7 @@ public class FilesystemQueryActionTest extends FilesystemActionTest QueryInput queryInput = new QueryInput(); QInstance instance = TestUtils.defineInstance(); - QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference() .withName(ValueUpshifter.class.getName()) .withCodeType(QCodeType.JAVA) diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java index e293c89a..e67936d5 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java @@ -24,7 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; import java.io.IOException; import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -44,7 +44,7 @@ class FilesystemBackendMetaDataTest ** Test that an instance can be serialized as expected *******************************************************************************/ @Test - public void testSerializingToJson() throws QInstanceValidationException + public void testSerializingToJson() throws QException { TestUtils.resetTestInstanceCounter(); QInstance qInstance = TestUtils.defineInstance(); @@ -62,7 +62,7 @@ class FilesystemBackendMetaDataTest ** Test that an instance can be deserialized as expected *******************************************************************************/ @Test - public void testDeserializingFromJson() throws IOException, QInstanceValidationException + public void testDeserializingFromJson() throws IOException, QException { QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); @@ -71,6 +71,8 @@ class FilesystemBackendMetaDataTest QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); assertThat(deserialized.getBackends()).usingRecursiveComparison() + // TODO seeing occassional flaps on this field - where it can be null 1 out of 10 runs... unclear why. + .ignoringFields("mock.backendType") .isEqualTo(qInstance.getBackends()); } -} \ No newline at end of file +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java index 3a43066a..33604e1b 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java @@ -198,7 +198,7 @@ public class BasicETLCleanupSourceFilesStepTest runBackendStepInput.setProcessName(qProcessMetaData.getName()); // runFunctionRequest.setRecords(records); runBackendStepInput.setSession(TestUtils.getMockSession()); - runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); runBackendStepInput.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); runBackendStepInput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); @@ -219,7 +219,7 @@ public class BasicETLCleanupSourceFilesStepTest private String getRandomFilePathPersonTable(QInstance qInstance) { FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS); - FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails(); + FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON).getBackendDetails(); String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath(); String filePath = tablePath + File.separator + UUID.randomUUID(); return filePath; diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStepTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStepTest.java new file mode 100644 index 00000000..2c71d2a2 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStepTest.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed; + + +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest; +import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic.BasicETLCollectSourceFileNamesStep; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for StreamedETLFilesystemBackendStep + *******************************************************************************/ +class StreamedETLFilesystemBackendStepTest extends FilesystemActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void runFullProcess() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + + RunProcessInput runProcessInput = new RunProcessInput(qInstance); + runProcessInput.setSession(TestUtils.getMockSession()); + runProcessInput.setProcessName(TestUtils.PROCESS_NAME_STREAMED_ETL); + + RunProcessOutput output = new RunProcessAction().execute(runProcessInput); + String sourceFilePaths = ValueUtils.getValueAsString(output.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS)); + assertThat(sourceFilePaths) + .contains("FILE-1.csv") + .contains("FILE-2.csv"); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java index ef84e292..3f790fa0 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -60,7 +59,7 @@ public class S3QueryActionTest extends BaseS3Test /******************************************************************************* ** *******************************************************************************/ - private QueryInput initQueryRequest() throws QInstanceValidationException + private QueryInput initQueryRequest() throws QException { QueryInput queryInput = new QueryInput(); queryInput.setInstance(TestUtils.defineInstance()); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java index 001fc6cb..39103602 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -22,9 +22,8 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; -import java.io.IOException; import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -44,7 +43,7 @@ class S3BackendMetaDataTest ** Test that an instance can be serialized as expected *******************************************************************************/ @Test - public void testSerializingToJson() throws QInstanceValidationException + public void testSerializingToJson() throws QException { TestUtils.resetTestInstanceCounter(); QInstance qInstance = TestUtils.defineInstance(); @@ -62,7 +61,7 @@ class S3BackendMetaDataTest ** Test that an instance can be deserialized as expected *******************************************************************************/ @Test - public void testDeserializingFromJson() throws IOException, QInstanceValidationException + public void testDeserializingFromJson() throws Exception { QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); @@ -71,6 +70,8 @@ class S3BackendMetaDataTest QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); assertThat(deserialized.getBackends()).usingRecursiveComparison() + // TODO seeing occassional flaps on this field - where it can be null 1 out of 10 runs... unclear why. + .ignoringFields("mock.backendType") .isEqualTo(qInstance.getBackends()); } -} \ No newline at end of file +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index 3c32d746..b9e76476 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -28,6 +28,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; @@ -92,7 +93,20 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte List outputRecords = new ArrayList<>(); rs.setRecords(outputRecords); - try(Connection connection = getConnection(insertInput)) + Connection connection; + boolean needToCloseConnection = false; + if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction) + { + LOG.debug("Using connection from insertInput [" + rdbmsTransaction.getConnection() + "]"); + connection = rdbmsTransaction.getConnection(); + } + else + { + connection = getConnection(insertInput); + needToCloseConnection = true; + } + + try { for(List page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE)) { @@ -130,6 +144,13 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte } } } + finally + { + if(needToCloseConnection) + { + connection.close(); + } + } return rs; } @@ -139,4 +160,28 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte } } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QBackendTransaction openTransaction(InsertInput insertInput) throws QException + { + try + { + LOG.info("Opening transaction"); + Connection connection = getConnection(insertInput); + connection.setAutoCommit(false); + System.out.println("Set connection [" + connection + "] to auto-commit false"); + + return (new RDBMSTransaction(connection)); + } + catch(Exception e) + { + throw new QException("Error opening transaction: " + e.getMessage(), e); + } + } + + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java new file mode 100644 index 00000000..529f69ff --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.actions; + + +import java.io.IOException; +import java.sql.Connection; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** RDBMS implementation of backend transaction. + ** + ** Stores a jdbc connection, which is set to autoCommit(false). + *******************************************************************************/ +public class RDBMSTransaction extends QBackendTransaction +{ + private static final Logger LOG = LogManager.getLogger(RDBMSTransaction.class); + + private Connection connection; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public RDBMSTransaction(Connection connection) + { + this.connection = connection; + } + + + + /******************************************************************************* + ** Getter for connection + ** + *******************************************************************************/ + public Connection getConnection() + { + return connection; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void commit() throws QException + { + try + { + RDBMSTransaction.LOG.info("Committing transaction"); + System.out.println("Calling commit on connection [" + connection + "]"); + connection.commit(); + RDBMSTransaction.LOG.info("Commit complete"); + } + catch(Exception e) + { + RDBMSTransaction.LOG.error("Error committing transaction", e); + throw new QException("Error committing transaction: " + e.getMessage(), e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void rollback() throws QException + { + try + { + RDBMSTransaction.LOG.info("Rolling back transaction"); + connection.rollback(); + RDBMSTransaction.LOG.info("Rollback complete"); + } + catch(Exception e) + { + RDBMSTransaction.LOG.error("Error rolling back transaction", e); + throw new QException("Error rolling back transaction: " + e.getMessage(), e); + } + } + + + + /******************************************************************************* + * Closes this stream and releases any system resources associated + * with it. If the stream is already closed then invoking this + * method has no effect. + * + *

As noted in {@link AutoCloseable#close()}, cases where the + * close may fail require careful attention. It is strongly advised + * to relinquish the underlying resources and to internally + * mark the {@code Closeable} as closed, prior to throwing + * the {@code IOException}. + * + * @throws IOException + * if an I/O error occurs + *******************************************************************************/ + @Override + public void close() + { + try + { + if(connection.isClosed()) + { + return; + } + + connection.close(); + } + catch(Exception e) + { + LOG.error("Error closing connection - possible jdbc connection leak", e); + } + } +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index be854f39..e5961b38 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -690,6 +690,11 @@ public class QueryManager bindParam(statement, index, l.intValue()); return (1); } + else if(value instanceof Double d) + { + bindParam(statement, index, d.doubleValue()); + return (1); + } else if(value instanceof String s) { bindParam(statement, index, s); @@ -860,6 +865,23 @@ public class QueryManager + /******************************************************************************* + * + *******************************************************************************/ + public static void bindParam(PreparedStatement statement, int index, Double value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DOUBLE); + } + else + { + statement.setDouble(index, value); + } + } + + + /******************************************************************************* * *******************************************************************************/ diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 2dddf8d6..b67241ee 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -251,11 +251,39 @@ public class QJavalinImplementation try { Map authenticationContext = new HashMap<>(); - authenticationContext.put(SESSION_ID_COOKIE_NAME, context.cookie(SESSION_ID_COOKIE_NAME)); + + ///////////////////////////////////////////////////////////////////////////////// + // look for a token in either the sessionId cookie, or an Authorization header // + ///////////////////////////////////////////////////////////////////////////////// + String sessionIdCookieValue = context.cookie(SESSION_ID_COOKIE_NAME); + if(StringUtils.hasContent(sessionIdCookieValue)) + { + authenticationContext.put(SESSION_ID_COOKIE_NAME, sessionIdCookieValue); + } + else + { + String authorizationHeaderValue = context.header("Authorization"); + if (authorizationHeaderValue != null) + { + String bearerPrefix = "Bearer "; + if(authorizationHeaderValue.startsWith(bearerPrefix)) + { + authorizationHeaderValue = authorizationHeaderValue.replaceFirst(bearerPrefix, ""); + } + authenticationContext.put(SESSION_ID_COOKIE_NAME, authorizationHeaderValue); + } + } + QSession session = authenticationModule.createSession(qInstance, authenticationContext); input.setSession(session); - context.cookie(SESSION_ID_COOKIE_NAME, session.getIdReference(), SESSION_COOKIE_AGE); + ///////////////////////////////////////////////////////////////////////////////// + // if we got a session id cookie in, then send it back with updated cookie age // + ///////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(sessionIdCookieValue)) + { + context.cookie(SESSION_ID_COOKIE_NAME, session.getIdReference(), SESSION_COOKIE_AGE); + } } catch(QAuthenticationException qae) {