QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Re-enabling sample project, now that it can use h2 database (so it can run & pass in CI); cleanup some warnings, and some initial IJ configs

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-28 15:16:22 -05:00
parent eac0f3b3a4
commit 3d1652c852
17 changed files with 267 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -30,3 +30,4 @@ target/
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
.DS_Store
*.swp

25
.idea/checkstyle-idea.xml generated
View File

@ -1,16 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CheckStyle-IDEA">
<option name="configuration">
<map>
<entry key="checkstyle-version" value="9.0" />
<entry key="copy-libs" value="false" />
<entry key="location-0" value="BUNDLED:(bundled):Sun Checks" />
<entry key="location-1" value="BUNDLED:(bundled):Google Checks" />
<entry key="scan-before-checkin" value="false" />
<entry key="scanscope" value="JavaOnly" />
<entry key="suppress-errors" value="false" />
</map>
<component name="CheckStyle-IDEA" serialisationVersion="2">
<checkstyleVersion>9.0.1</checkstyleVersion>
<scanScope>JavaOnly</scanScope>
<suppressErrors>true</suppressErrors>
<option name="thirdPartyClasspath" />
<option name="activeLocationIds">
<option value="8dbc1112-3885-409e-8579-2aacdaedfb53" />
</option>
<option name="locations">
<list>
<ConfigurationLocation id="bundled-sun-checks" type="BUNDLED" scope="All" description="Sun Checks">(bundled)</ConfigurationLocation>
<ConfigurationLocation id="bundled-google-checks" type="BUNDLED" scope="All" description="Google Checks">(bundled)</ConfigurationLocation>
<ConfigurationLocation id="8dbc1112-3885-409e-8579-2aacdaedfb53" type="PROJECT_RELATIVE" scope="All" description="Kingsrook">$PROJECT_DIR$/checkstyle.xml</ConfigurationLocation>
</list>
</option>
</component>
</project>

1
.idea/compiler.xml generated
View File

@ -22,6 +22,7 @@
<module name="qqq-backend-module-rdbms" options="-Xlint:unchecked" />
<module name="qqq-middleware-javalin" options="-Xlint:unchecked" />
<module name="qqq-middleware-picocli" options="-Xlint:unchecked" />
<module name="qqq-parent-project" options="" />
<module name="qqq-sample-project" options="-Xlint:unchecked" />
</option>
</component>

6
.idea/inspectionProfiles/Project_Default.xml generated Normal file
View File

@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="JavadocLinkAsPlainText" enabled="false" level="WARNING" enabled_by_default="false" />
</profile>
</component>

1
.idea/misc.xml generated
View File

@ -5,6 +5,7 @@
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/qqq-sample-project/pom.xml" />
</list>
</option>
</component>

18
.idea/runConfigurations/All_in_qqq_sample_project.xml generated Normal file
View File

@ -0,0 +1,18 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="All in qqq-sample-project" type="JUnit" factoryName="JUnit" nameIsGenerated="true">
<module name="qqq-sample-project" />
<extension name="coverage" runner="jacoco">
<pattern>
<option name="PATTERN" value="com.kingsrook.sampleapp.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<option name="PACKAGE_NAME" value="" />
<option name="MAIN_CLASS_NAME" value="" />
<option name="METHOD_NAME" value="" />
<option name="TEST_OBJECT" value="package" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>

16
.idea/runConfigurations/Sample_Javalin.xml generated Normal file
View File

@ -0,0 +1,16 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Sample Javalin" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="com.kingsrook.sampleapp.SampleJavalinServer" />
<module name="qqq-sample-project" />
<option name="WORKING_DIRECTORY" value="$MODULE_WORKING_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.kingsrook.sampleapp.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>

2
pom.xml
View File

@ -34,7 +34,7 @@
<module>qqq-backend-module-filesystem</module>
<module>qqq-middleware-picocli</module>
<module>qqq-middleware-javalin</module>
<!-- <module>qqq-sample-project</module> -->
<module>qqq-sample-project</module>
</modules>
<properties>

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/DeserializerUtils.java
View File

@ -225,7 +225,7 @@ public class DeserializerUtils
// otherwise, either find some jackson annotation that makes sense, and apply it to the setter method, //
// or if no jackson annotation is right, then come up with annotation of our own. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
method.invoke(output, reflectivelyDeserialize((Class) parameterType, (TreeNode) value));
method.invoke(output, reflectivelyDeserialize((Class<?>) parameterType, (TreeNode) value));
}
}
catch(IllegalAccessException | InvocationTargetException | NoSuchMethodException | IOException e)

8
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java
View File

@ -62,7 +62,7 @@ public class BasicETLProcess
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)));
.withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)));
QStepMetaData transformFunction = new QBackendStepMetaData()
.withName(FUNCTION_NAME_TRANSFORM)
@ -71,8 +71,8 @@ public class BasicETLProcess
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)));
.withField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING))
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)));
QStepMetaData loadFunction = new QBackendStepMetaData()
.withName(FUNCTION_NAME_LOAD)
@ -81,7 +81,7 @@ public class BasicETLProcess
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)))
.withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)))
.withOutputMetaData(new QFunctionOutputMetaData()
.addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));

6
qqq-sample-project/pom.xml
View File

@ -65,6 +65,12 @@
<artifactId>qqq-middleware-picocli</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>2.1.212</version>
<scope>test</scope>
</dependency>
<!-- 3rd party deps specifically for this module -->
<!-- none at this time -->

20
qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java
View File

@ -40,6 +40,8 @@ public class SampleJavalinServer
private QInstance qInstance;
private Javalin javalinService;
/*******************************************************************************
@ -62,13 +64,13 @@ public class SampleJavalinServer
qInstance = SampleMetaDataProvider.defineInstance();
QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance);
Javalin service = Javalin.create(config ->
javalinService = Javalin.create(config ->
{
// todo - not all!!
config.enableCorsForAllOrigins();
}).start(PORT);
service.routes(qJavalinImplementation.getRoutes());
service.after(ctx ->
javalinService.routes(qJavalinImplementation.getRoutes());
javalinService.after(ctx ->
ctx.res.setHeader("Access-Control-Allow-Origin", "http://localhost:3000"));
}
catch(Exception e)
@ -77,4 +79,16 @@ public class SampleJavalinServer
}
}
/*******************************************************************************
**
*******************************************************************************/
public void stopJavalinServer()
{
if(javalinService != null)
{
javalinService.stop();
}
}
}

30
qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java
View File

@ -59,7 +59,9 @@ import io.github.cdimascio.dotenv.Dotenv;
*******************************************************************************/
public class SampleMetaDataProvider
{
public static final String MYSQL_BACKEND_NAME = "mysql";
public static boolean USE_MYSQL = false;
public static final String RDBMS_BACKEND_NAME = "rdbms";
public static final String FILESYSTEM_BACKEND_NAME = "filesystem";
public static final String AUTH0_AUTHENTICATION_MODULE_NAME = "auth0";
@ -88,7 +90,7 @@ public class SampleMetaDataProvider
QInstance qInstance = new QInstance();
qInstance.setAuthentication(defineAuthentication());
qInstance.addBackend(defineMysqlBackend());
qInstance.addBackend(defineRdbmsBackend());
qInstance.addBackend(defineFilesystemBackend());
qInstance.addTable(defineTableCarrier());
qInstance.addTable(defineTablePerson());
@ -119,17 +121,29 @@ public class SampleMetaDataProvider
/*******************************************************************************
**
*******************************************************************************/
public static QBackendMetaData defineMysqlBackend()
public static RDBMSBackendMetaData defineRdbmsBackend()
{
if(USE_MYSQL)
{
Dotenv dotenv = Dotenv.configure().load();
return new RDBMSBackendMetaData()
.withName(RDBMS_BACKEND_NAME)
.withVendor("mysql")
.withHostName("127.0.0.1")
.withPort(3306)
.withDatabaseName("qqq")
.withUsername("root")
.withPassword(dotenv.get("RDBMS_PASSWORD"))
.withName(MYSQL_BACKEND_NAME);
.withPassword(dotenv.get("RDBMS_PASSWORD"));
}
else
{
return (new RDBMSBackendMetaData()
.withName(RDBMS_BACKEND_NAME)
.withVendor("h2")
.withHostName("mem")
.withDatabaseName("test_database")
.withUsername("sa"));
}
}
@ -153,7 +167,7 @@ public class SampleMetaDataProvider
{
QTableMetaData table = new QTableMetaData();
table.setName("carrier");
table.setBackendName(MYSQL_BACKEND_NAME);
table.setBackendName(RDBMS_BACKEND_NAME);
table.setPrimaryKeyField("id");
table.addField(new QFieldMetaData("id", QFieldType.INTEGER));
@ -182,7 +196,7 @@ public class SampleMetaDataProvider
return new QTableMetaData()
.withName("person")
.withLabel("Person")
.withBackendName(MYSQL_BACKEND_NAME)
.withBackendName(RDBMS_BACKEND_NAME)
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
@ -443,7 +457,7 @@ public class SampleMetaDataProvider
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(ThrowerStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER))));
.withField(new QFieldMetaData(ThrowerStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER))));
}
}

46
qqq-sample-project/src/main/java/com/kingsrook/sampleapp/customizers/FedExInvoiceCSVSyntaxFixer.java
View File

@ -1,46 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.sampleapp.customizers;
import java.util.function.Function;
/*******************************************************************************
** The fedex invoice files - they're CSV, but they have a handful of values
** that we can't read, because they have an extra quote (") character within
** a field.
**
** It always looks like: {"u
** This function fixes it by stripping the " out of the middle, to just be: {u
*******************************************************************************/
public class FedExInvoiceCSVSyntaxFixer implements Function<String, String>
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public String apply(String s)
{
return (s.replaceAll("\\{\"u", "{u"));
}
}

24
qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleJavalinServerTest.java Normal file
View File

@ -0,0 +1,24 @@
package com.kingsrook.sampleapp;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/*******************************************************************************
** Unit test for com.kingsrook.sampleapp.SampleJavalinServer
*******************************************************************************/
class SampleJavalinServerTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testStartStop()
{
SampleJavalinServer sampleJavalinServer = new SampleJavalinServer();
sampleJavalinServer.startJavalinServer();
sampleJavalinServer.stopJavalinServer();
}
}

67
qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java
View File

@ -24,9 +24,13 @@ package com.kingsrook.sampleapp;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Connection;
import java.util.List;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
@ -37,10 +41,16 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -50,6 +60,40 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class SampleMetaDataProviderTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
void beforeEach() throws Exception
{
primeTestDatabase("prime-test-database.sql");
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("unchecked")
private void primeTestDatabase(String sqlFileName) throws Exception
{
ConnectionManager connectionManager = new ConnectionManager();
try(Connection connection = connectionManager.getConnection(SampleMetaDataProvider.defineRdbmsBackend()))
{
InputStream primeTestDatabaseSqlStream = SampleMetaDataProviderTest.class.getResourceAsStream("/" + sqlFileName);
Assert.assertNotNull(primeTestDatabaseSqlStream);
List<String> lines = (List<String>) IOUtils.readLines(primeTestDatabaseSqlStream);
lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList();
String joinedSQL = String.join("\n", lines);
for(String sql : joinedSQL.split(";"))
{
QueryManager.executeUpdate(connection, sql);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -77,6 +121,9 @@ class SampleMetaDataProviderTest
/*******************************************************************************
**
*******************************************************************************/
private File copyTestFileToRandomNameUnderTable(QTableMetaData fedExTable) throws IOException
{
File destinationDir = new File(SampleMetaDataProvider.defineFilesystemBackend().getBasePath() + File.separator +
@ -121,4 +168,24 @@ class SampleMetaDataProviderTest
assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testThrowProcess() throws Exception
{
QInstance qInstance = SampleMetaDataProvider.defineInstance();
RunProcessInput request = new RunProcessInput(qInstance);
request.setSession(new QSession());
request.setProcessName(SampleMetaDataProvider.PROCESS_NAME_SIMPLE_THROW);
request.addValue(SampleMetaDataProvider.ThrowerStep.FIELD_SLEEP_MILLIS, 10);
assertThrows(QException.class, () ->
{
new RunProcessAction().execute(request);
});
}
}

60
qqq-sample-project/src/test/resources/prime-test-database.sql Normal file
View File

@ -0,0 +1,60 @@
--
-- QQQ - Low-code Application Framework for Engineers.
-- Copyright (C) 2021-2022. Kingsrook, LLC
-- 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
-- contact@kingsrook.com
-- https://github.com/Kingsrook/
--
-- This program is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as
-- published by the Free Software Foundation, either version 3 of the
-- License, or (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
--
DROP TABLE IF EXISTS person;
CREATE TABLE person
(
id INT AUTO_INCREMENT primary key ,
create_date TIMESTAMP DEFAULT now(),
modify_date TIMESTAMP DEFAULT now(),
first_name VARCHAR(80) NOT NULL,
last_name VARCHAR(80) NOT NULL,
birth_date DATE,
email VARCHAR(250) NOT NULL
);
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com');
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com');
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com');
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com');
INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com');
DROP TABLE IF EXISTS carrier;
CREATE TABLE carrier
(
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(80) NOT NULL,
company_code VARCHAR(80) NOT NULL,
service_level VARCHAR(80) NOT NULL
);
INSERT INTO carrier (id, name, company_code, service_level) VALUES (1, 'UPS Ground', 'UPS', 'G');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (2, 'UPS 2Day', 'UPS', '2');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (3, 'UPS International', 'UPS', 'I');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (4, 'Fedex Ground', 'FEDEX', 'G');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (5, 'Fedex Next Day', 'UPS', '1');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (6, 'Will Call', 'WILL_CALL', 'W');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (7, 'USPS Priority', 'USPS', '1');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (8, 'USPS Super Slow', 'USPS', '4');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (9, 'USPS Super Fast', 'USPS', '0');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (10, 'DHL International', 'DHL', 'I');
INSERT INTO carrier (id, name, company_code, service_level) VALUES (11, 'GSO', 'GSO', 'G');