diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/MetaDataProducerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/MetaDataProducerInterface.java new file mode 100644 index 00000000..45bd2ca6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/MetaDataProducerInterface.java @@ -0,0 +1,78 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; + + +/******************************************************************************* + ** Interface for classes that know how to produce meta data objects. Useful with + ** MetaDataProducerHelper, to put point at a package full of these, and populate + ** your whole QInstance. + ** + ** See also MetaDataProducer - an implementer of this interface, which actually + ** came first, and is fine to extend if producing a meta-data class is all your + ** clas means to do (nice and "Single-responsibility principle"). + ** + ** But, in some applications you may want to, for example, have one class that + ** defines a process step, and also produces the meta-data for that process, so + ** your whole process can just be one class - so then just have your step class + ** implement this interface. or, same idea for a QRecordEntity that provides + ** its own TableMetaData. + *******************************************************************************/ +public interface MetaDataProducerInterface +{ + int DEFAULT_SORT_ORDER = 500; + + + /******************************************************************************* + ** Produce the metaData object. Generally, you don't want to add it to the instance + ** yourself - but the instance is there in case you need it to get other metaData. + *******************************************************************************/ + T produce(QInstance qInstance) throws QException; + + + /******************************************************************************* + ** In case this producer needs to run before (or after) others, this method + ** can control influence that (e.g., if used by MetaDataProducerHelper). + ** + ** Smaller values run first. + *******************************************************************************/ + default int getSortOrder() + { + return (DEFAULT_SORT_ORDER); + } + + + /******************************************************************************* + ** turn this producer on or off - e.g., maybe based on an env value. + ** + *******************************************************************************/ + default boolean isEnabled() + { + return (true); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java index 1b3e2148..4207a132 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java @@ -22,7 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata; -import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; /******************************************************************************* @@ -30,29 +30,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; ** MetaDataProducerHelper, to put point at a package full of these, and populate ** your whole QInstance. *******************************************************************************/ -public abstract class MetaDataProducer +public abstract class MetaDataProducer implements MetaDataProducerInterface { - public static final int DEFAULT_SORT_ORDER = 500; - - - - /******************************************************************************* - ** Produce the metaData object. Generally, you don't want to add it to the instance - ** yourself - but the instance is there in case you need it to get other metaData. - *******************************************************************************/ - public abstract T produce(QInstance qInstance) throws QException; - - - - /******************************************************************************* - ** In case this producer needs to run before (or after) others, this method - ** can control influence that (e.g., if used by MetaDataProducerHelper). - ** - ** Smaller values run first. - *******************************************************************************/ - public int getSortOrder() - { - return (DEFAULT_SORT_ORDER); - } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index 7769c329..7a52a890 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -30,6 +30,7 @@ import java.util.Comparator; import java.util.List; import com.google.common.reflect.ClassPath; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -44,7 +45,7 @@ public class MetaDataProducerHelper /******************************************************************************* - ** Recursively find all classes in the given package, that extend MetaDataProducer, + ** Recursively find all classes in the given package, that implement MetaDataProducerInterface ** run them, and add their output to the given qInstance. ** ** Note - they'll be sorted by the sortOrder they provide. @@ -54,8 +55,8 @@ public class MetaDataProducerHelper //////////////////////////////////////////////////////////////////////// // find all the meta data producer classes in (and under) the package // //////////////////////////////////////////////////////////////////////// - List> classesInPackage = getClassesInPackage(packageName); - List> producers = new ArrayList<>(); + List> classesInPackage = getClassesInPackage(packageName); + List> producers = new ArrayList<>(); for(Class aClass : classesInPackage) { try @@ -65,22 +66,29 @@ public class MetaDataProducerHelper continue; } - for(Constructor constructor : aClass.getConstructors()) + if(MetaDataProducerInterface.class.isAssignableFrom(aClass)) { - if(constructor.getParameterCount() == 0) + boolean foundValidConstructor = false; + for(Constructor constructor : aClass.getConstructors()) { - Object o = constructor.newInstance(); - if(o instanceof MetaDataProducer metaDataProducer) + if(constructor.getParameterCount() == 0) { - producers.add(metaDataProducer); + Object o = constructor.newInstance(); + producers.add((MetaDataProducerInterface) o); + foundValidConstructor = true; + break; } - break; + } + + if(!foundValidConstructor) + { + LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", aClass.getSimpleName())); } } } catch(Exception e) { - LOG.info("Error adding metaData from producer", e, logPair("producer", aClass.getSimpleName())); + LOG.warn("Error evaluating a possible meta-data producer class", e, logPair("class", aClass.getSimpleName())); } } @@ -89,8 +97,8 @@ public class MetaDataProducerHelper // after all other types (as apps often try to get other types from the instance) // //////////////////////////////////////////////////////////////////////////////////////////// producers.sort(Comparator - .comparing((MetaDataProducer p) -> p.getSortOrder()) - .thenComparing((MetaDataProducer p) -> + .comparing((MetaDataProducerInterface p) -> p.getSortOrder()) + .thenComparing((MetaDataProducerInterface p) -> { try { @@ -110,22 +118,29 @@ public class MetaDataProducerHelper } })); - ////////////////////////////////////////////////////////////// - // execute each one, adding their meta data to the instance // - ////////////////////////////////////////////////////////////// - for(MetaDataProducer producer : producers) + /////////////////////////////////////////////////////////////////////////// + // execute each one (if enabled), adding their meta data to the instance // + /////////////////////////////////////////////////////////////////////////// + for(MetaDataProducerInterface producer : producers) { - try + if(producer.isEnabled()) { - TopLevelMetaDataInterface metaData = producer.produce(instance); - if(metaData != null) + try { - metaData.addSelfToInstance(instance); + TopLevelMetaDataInterface metaData = producer.produce(instance); + if(metaData != null) + { + metaData.addSelfToInstance(instance); + } + } + catch(Exception e) + { + LOG.warn("error executing metaDataProducer", logPair("producer", producer.getClass().getSimpleName()), e); } } - catch(Exception e) + else { - LOG.warn("error executing metaDataProducer", logPair("producer", producer.getClass().getSimpleName()), e); + LOG.debug("Not using producer which is not enabled", logPair("producer", producer.getClass().getSimpleName())); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelperTest.java index 8c96b8ab..ab1ae66f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelperTest.java @@ -23,8 +23,14 @@ package com.kingsrook.qqq.backend.core.model.metadata; import java.io.IOException; +import com.kingsrook.qqq.backend.core.model.metadata.producers.TestAbstractMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.producers.TestDisabledMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.producers.TestImplementsMetaDataProducer; import com.kingsrook.qqq.backend.core.model.metadata.producers.TestMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.producers.TestNoInterfacesExtendsObject; +import com.kingsrook.qqq.backend.core.model.metadata.producers.TestNoValidConstructorMetaDataProducer; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -43,6 +49,11 @@ class MetaDataProducerHelperTest QInstance qInstance = new QInstance(); MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, "com.kingsrook.qqq.backend.core.model.metadata.producers"); assertTrue(qInstance.getTables().containsKey(TestMetaDataProducer.NAME)); + assertTrue(qInstance.getTables().containsKey(TestImplementsMetaDataProducer.NAME)); + assertFalse(qInstance.getTables().containsKey(TestNoValidConstructorMetaDataProducer.NAME)); + assertFalse(qInstance.getTables().containsKey(TestNoInterfacesExtendsObject.NAME)); + assertFalse(qInstance.getTables().containsKey(TestAbstractMetaDataProducer.NAME)); + assertFalse(qInstance.getTables().containsKey(TestDisabledMetaDataProducer.NAME)); } } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestAbstractMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestAbstractMetaDataProducer.java new file mode 100644 index 00000000..041c1473 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestAbstractMetaDataProducer.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.producers; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class TestAbstractMetaDataProducer extends MetaDataProducer +{ + public static final String NAME = "TestAbstractMetaDataProducer"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + return new QTableMetaData().withName(NAME); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestDisabledMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestDisabledMetaDataProducer.java new file mode 100644 index 00000000..5d2c3be7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestDisabledMetaDataProducer.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.producers; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestDisabledMetaDataProducer implements MetaDataProducerInterface +{ + public static final String NAME = "Disabled"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean isEnabled() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + return new QTableMetaData().withName(NAME); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestImplementsMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestImplementsMetaDataProducer.java new file mode 100644 index 00000000..14ce9359 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestImplementsMetaDataProducer.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.producers; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestImplementsMetaDataProducer implements MetaDataProducerInterface +{ + public static final String NAME = "BuiltByProducerImplementingInterface"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + return new QTableMetaData().withName(NAME); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestNoInterfacesExtendsObject.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestNoInterfacesExtendsObject.java new file mode 100644 index 00000000..e7cc6a0f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestNoInterfacesExtendsObject.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.producers; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestNoInterfacesExtendsObject +{ + public static final String NAME = "TestNoInterfacesExtendsObject"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData produce(QInstance qInstance) throws QException + { + return new QTableMetaData().withName(NAME); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestNoValidConstructorMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestNoValidConstructorMetaDataProducer.java new file mode 100644 index 00000000..94463a54 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestNoValidConstructorMetaDataProducer.java @@ -0,0 +1,58 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.producers; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestNoValidConstructorMetaDataProducer extends MetaDataProducer +{ + public static final String NAME = "NoValidConstructor"; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TestNoValidConstructorMetaDataProducer(boolean b) + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + return new QTableMetaData().withName(NAME); + } +} diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index bdcb5fe6..a14b23b1 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -81,6 +81,7 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; @@ -1048,7 +1049,7 @@ public class BaseAPIActionUtil ////////////////////////////////////////////////////// // make sure to use closeable client to avoid leaks // ////////////////////////////////////////////////////// - try(CloseableHttpClient httpClient = HttpClientBuilder.create().build()) + try(CloseableHttpClient httpClient = buildHttpClient()) { //////////////////////////////////////////////////////////// // call utility methods that populate data in the request // @@ -1153,6 +1154,25 @@ public class BaseAPIActionUtil + /******************************************************************************* + ** Build the default HttpClient used by the makeRequest method + *******************************************************************************/ + protected CloseableHttpClient buildHttpClient() + { + /////////////////////////////////////////////////////////////////////////////////////// + // do we want this?? .setConnectionManager(new PoolingHttpClientConnectionManager()) // + // needs some good scrutiny. // + /////////////////////////////////////////////////////////////////////////////////////// + return HttpClientBuilder.create() + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectTimeout(getConnectionTimeoutMillis()) + .setConnectionRequestTimeout(getConnectionRequestTimeoutMillis()) + .setSocketTimeout(getSocketTimeoutMillis()).build()) + .build(); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -1439,6 +1459,51 @@ public class BaseAPIActionUtil + /******************************************************************************* + ** For the HttpClientBuilder RequestConfig, specify its ConnectionTimeout. See + ** - https://www.baeldung.com/httpclient-timeout + ** - https://hc.apache.org/httpcomponents-client-5.1.x/current/httpclient5/apidocs/org/apache/hc/client5/http/config/RequestConfig.Builder.html + *******************************************************************************/ + protected int getConnectionTimeoutMillis() + { + ////////////// + // 1 minute // + ////////////// + return (60 * 1000); + } + + + + /******************************************************************************* + ** For the HttpClientBuilder RequestConfig, specify its ConnectionRequestTimeout. See + ** - https://www.baeldung.com/httpclient-timeout + ** - https://hc.apache.org/httpcomponents-client-5.1.x/current/httpclient5/apidocs/org/apache/hc/client5/http/config/RequestConfig.Builder.html + *******************************************************************************/ + protected int getConnectionRequestTimeoutMillis() + { + ////////////// + // 1 minute // + ////////////// + return (60 * 1000); + } + + + + /******************************************************************************* + ** For the HttpClientBuilder RequestConfig, specify its ConnectionRequestTimeout. See + ** - https://www.baeldung.com/httpclient-timeout + ** - https://hc.apache.org/httpcomponents-client-5.1.x/current/httpclient5/apidocs/org/apache/hc/client5/http/config/RequestConfig.Builder.html + *******************************************************************************/ + protected int getSocketTimeoutMillis() + { + /////////////// + // 3 minutes // + /////////////// + return (3 * 60 * 1000); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java b/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java index 25b1af88..60923ea5 100644 --- a/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java +++ b/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java @@ -22,11 +22,18 @@ package com.kingsrook.qqq.backend.module.api.actions; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.PrintWriter; import java.io.Serializable; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; @@ -36,6 +43,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; @@ -82,6 +90,8 @@ import static org.junit.jupiter.api.Assertions.fail; *******************************************************************************/ class BaseAPIActionUtilTest extends BaseTest { + private static final QLogger LOG = QLogger.getLogger(BaseAPIActionUtilTest.class); + private static MockApiUtilsHelper mockApiUtilsHelper = new MockApiUtilsHelper(); @@ -822,6 +832,108 @@ class BaseAPIActionUtilTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTimeouts() throws QException + { + ShortTimeoutActionUtil shortTimeoutActionUtil = new ShortTimeoutActionUtil(); + shortTimeoutActionUtil.setBackendMetaData((APIBackendMetaData) QContext.getQInstance().getBackend(TestUtils.MOCK_BACKEND_NAME)); + + ///////////////////////////////////////////////////////////// + // make sure we work correctly with a large enough timeout // + ///////////////////////////////////////////////////////////// + { + startSimpleHttpServer(8888); + HttpGet request = new HttpGet("http://localhost:8888"); + shortTimeoutActionUtil.setTimeoutMillis(3000); + + shortTimeoutActionUtil.makeRequest(QContext.getQInstance().getTable(TestUtils.MOCK_TABLE_NAME), request); + } + + //////////////////////////////////////////////// + // make sure we fail with a too-small timeout // + //////////////////////////////////////////////// + { + startSimpleHttpServer(8889); + HttpGet request = new HttpGet("http://localhost:8889"); + shortTimeoutActionUtil.setTimeoutMillis(1); + + assertThatThrownBy(() -> shortTimeoutActionUtil.makeRequest(QContext.getQInstance().getTable(TestUtils.MOCK_TABLE_NAME), request)) + .hasRootCauseInstanceOf(SocketTimeoutException.class) + .rootCause().hasMessageContaining("timed out"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void startSimpleHttpServer(int port) + { + Executors.newSingleThreadExecutor().submit(() -> + { + LOG.info("Listening on " + port); + try(ServerSocket serverSocket = new ServerSocket(port)) + { + Socket clientSocket = serverSocket.accept(); + PrintWriter out = new PrintWriter(clientSocket.getOutputStream(), true); + BufferedReader in = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); + String greeting = in.readLine(); + LOG.info("Read: " + greeting); + SleepUtils.sleep(1, TimeUnit.SECONDS); + out.println("HTTP/1.1 200 OK"); + out.close(); + clientSocket.close(); + } + catch(Exception e) + { + LOG.info("Exception in simple http server", e); + } + }); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // give time for the thread w/ the listening socket to start before returning control to the thread that's going to try to connect to it // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + SleepUtils.sleep(100, TimeUnit.MILLISECONDS); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static class ShortTimeoutActionUtil extends BaseAPIActionUtil + { + private int timeoutMillis = 1; + + + + /******************************************************************************* + ** Setter for timeoutMillis + ** + *******************************************************************************/ + public void setTimeoutMillis(int timeoutMillis) + { + this.timeoutMillis = timeoutMillis; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected int getSocketTimeoutMillis() + { + return (timeoutMillis); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 36d23362..b93b52f2 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -64,6 +65,19 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf private ActionTimeoutHelper actionTimeoutHelper; + private static boolean mysqlResultSetOptimizationEnabled = false; + + static + { + try + { + mysqlResultSetOptimizationEnabled = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.rdbms.mysql.resultSetOptimizationEnabled", "QQQ_RDBMS_MYSQL_RESULT_SET_OPTIMIZATION_ENABLED", false); + } + catch(Exception e) + { + LOG.warn("Error reading property/env for mysqlResultSetOptimizationEnabled", e); + } + } /******************************************************************************* @@ -342,22 +356,19 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf *******************************************************************************/ private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException { - PreparedStatement statement; - if(connection.getClass().getName().startsWith("com.mysql")) + if(mysqlResultSetOptimizationEnabled && connection.getClass().getName().startsWith("com.mysql")) { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html // // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // // with this change, we start to get results immediately, and the total runtime also seems lower... // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); statement.setFetchSize(Integer.MIN_VALUE); + return (statement); } - else - { - statement = connection.prepareStatement(sql); - } - return (statement); + + return (connection.prepareStatement(sql)); }