Checkpoint - working with initial version 0.1.0-SNAPSHOT's

.circleci/config.yml

@@ -1,3 +1,5 @@
+## Deviations from qqq-java library standard circleci config:
+## - In this project, we just always run a test, never a deploy (at this time).
 version: 2.1
 
 executors:
@@ -59,20 +61,4 @@ workflows:
     jobs:
       - mvn_test:
           context: [ qqq-maven-registry-credentials, kingsrook-slack ]
-## just doing mvn test for all branches in this repo at this time
+
-##         filters:
-##           branches:
-##             ignore: /dev/
-##           tags:
-##             ignore: /version-.*/
-##
-## deploy:
-##   jobs:
-##     - mvn_deploy:
-##         context: [ qqq-maven-registry-credentials, kingsrook-slack ]
-##         filters:
-##           branches:
-##             only: /dev/
-##           tags:
-##             only: /version-.*/
-##

.gitignore

@@ -28,3 +28,5 @@ target/
 
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
+
+.DS_Store

checkstyle.xml

@@ -46,6 +46,7 @@
    -->
 
    <module name="TreeWalker">
+      <module name="SuppressWarningsHolder"/>
       <module name="OuterTypeFilename"/>
       <module name="IllegalTokenText">
          <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
@@ -171,7 +172,7 @@
          <property name="caseIndent" value="3"/>
          <property name="throwsIndent" value="6"/>
          <property name="lineWrappingIndentation" value="3"/>
-         <property name="arrayInitIndent" value="2"/>
+         <property name="arrayInitIndent" value="6"/>
       </module>
       <!--
       <module name="AbbreviationAsWordInName">
@@ -260,4 +261,5 @@
       <module name="MissingOverride"/>
 
    </module>
+   <module name="SuppressWarningsFilter"/>
 </module>

pom.xml

@@ -27,7 +27,7 @@
 
    <groupId>com.kingsrook.qqq</groupId>
    <artifactId>qqq-sample-project</artifactId>
-   <version>0.0.0-SNAPSHOT</version>
+   <version>0.1.0-SNAPSHOT</version>
 
    <properties>
       <!-- props specifically to this module -->
@@ -47,25 +47,31 @@
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-core</artifactId>
-         <version>0.0.0-SNAPSHOT</version>
+         <version>0.1.0-SNAPSHOT</version>
          <scope>compile</scope>
       </dependency>
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-module-rdbms</artifactId>
-         <version>0.0.0-SNAPSHOT</version>
+         <version>0.1.0-SNAPSHOT</version>
+         <scope>compile</scope>
+      </dependency>
+      <dependency>
+         <groupId>com.kingsrook.qqq</groupId>
+         <artifactId>qqq-backend-module-filesystem</artifactId>
+         <version>0.1.0-SNAPSHOT</version>
          <scope>compile</scope>
       </dependency>
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-middleware-javalin</artifactId>
-         <version>0.0.0-SNAPSHOT</version>
+         <version>0.1.0-SNAPSHOT</version>
          <scope>compile</scope>
       </dependency>
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-middleware-picocli</artifactId>
-         <version>0.0.0-SNAPSHOT</version>
+         <version>0.1.0-SNAPSHOT</version>
          <scope>compile</scope>
       </dependency>
 
@@ -98,7 +104,19 @@
    <build>
       <plugins>
          <!-- plugins specifically for this module -->
-         <!-- none at this time -->
+         <plugin>
+            <artifactId>maven-assembly-plugin</artifactId>
+            <configuration>
+               <descriptorRefs>
+                  <descriptorRef>jar-with-dependencies</descriptorRef>
+               </descriptorRefs>
+               <archive>
+                  <manifest>
+                     <mainClass>com.kingsrook.sampleapp.SampleCli</mainClass>
+                  </manifest>
+               </archive>
+            </configuration>
+         </plugin>
 
          <!-- Common plugins for all qqq modules -->
          <plugin>

src/main/java/com/kingsrook/sampleapp/SampleCli.java

@@ -46,9 +46,18 @@ public class SampleCli
     *******************************************************************************/
    private void run(String[] args)
    {
-      QInstance qInstance = SampleMetaDataProvider.defineInstance();
+      try
-      QPicoCliImplementation qPicoCliImplementation = new QPicoCliImplementation(qInstance);
+      {
-      int exitCode = qPicoCliImplementation.runCli("my-sample-cli", args);
+         QInstance              qInstance              = SampleMetaDataProvider.defineInstance();
-      System.exit(exitCode);
+         QPicoCliImplementation qPicoCliImplementation = new QPicoCliImplementation(qInstance);
+         int                    exitCode               = qPicoCliImplementation.runCli("my-sample-cli", args);
+         System.exit(exitCode);
+      }
+      catch(Exception e)
+      {
+         e.printStackTrace();
+         System.exit(-1);
+      }
    }
+
 }

src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java

@@ -57,17 +57,24 @@ public class SampleJavalinServer
     *******************************************************************************/
    public void startJavalinServer()
    {
-      qInstance = SampleMetaDataProvider.defineInstance();
+      try
-
-      QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance);
-      Javalin service = Javalin.create(config ->
       {
-         // todo - not all!!
+         qInstance = SampleMetaDataProvider.defineInstance();
-         config.enableCorsForAllOrigins();
+
-      }).start(PORT);
+         QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance);
-      service.routes(qJavalinImplementation.getRoutes());
+         Javalin service = Javalin.create(config ->
-      service.after(ctx ->
+         {
-         ctx.res.setHeader("Access-Control-Allow-Origin", "http://localhost:3000"));
+            // todo - not all!!
+            config.enableCorsForAllOrigins();
+         }).start(PORT);
+         service.routes(qJavalinImplementation.getRoutes());
+         service.after(ctx ->
+            ctx.res.setHeader("Access-Control-Allow-Origin", "http://localhost:3000"));
+      }
+      catch(Exception e)
+      {
+         LOG.error("Failed to start javalin server.  See stack trace for details.", e);
+      }
    }
 
 }

src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java

@@ -22,12 +22,28 @@
 package com.kingsrook.sampleapp;
 
 
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.interfaces.mock.MockBackendStep;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
 
 
 /*******************************************************************************
@@ -35,20 +51,27 @@ import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
  *******************************************************************************/
 public class SampleMetaDataProvider
 {
-   private static final String BACKEND_NAME = "default";
+   public static final String MYSQL_BACKEND_NAME      = "mysql";
+   public static final String FILESYSTEM_BACKEND_NAME = "filesystem";
+   public static final String PROCESS_NAME_GREET      = "greet";
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static QInstance defineInstance()
+   public static QInstance defineInstance() throws QException
    {
       QInstance qInstance = new QInstance();
+
       qInstance.setAuthentication(SampleMetaDataProvider.defineAuthentication());
-      qInstance.addBackend(SampleMetaDataProvider.defineBackend());
+      qInstance.addBackend(SampleMetaDataProvider.defineMysqlBackend());
+      qInstance.addBackend(SampleMetaDataProvider.defineFilesystemBackend());
       qInstance.addTable(SampleMetaDataProvider.defineTableCarrier());
       qInstance.addTable(SampleMetaDataProvider.defineTablePerson());
+      qInstance.addTable(SampleMetaDataProvider.defineTableCityFile());
+      qInstance.addProcess(SampleMetaDataProvider.defineProcessGreetPeople());
+
       return (qInstance);
    }
 
@@ -69,18 +92,28 @@ public class SampleMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static QBackendMetaData defineBackend()
+   public static QBackendMetaData defineMysqlBackend()
    {
-      QBackendMetaData backend = new QBackendMetaData();
+      return new RDBMSBackendMetaData()
-      backend.setName(BACKEND_NAME);
+         .withVendor("mysql")
-      backend.setType("rdbms");
+         .withHostName("127.0.0.1")
-      backend.setValue("vendor", "mysql");
+         .withPort(3306)
-      backend.setValue("hostName", "127.0.0.1");
+         .withDatabaseName("sample_project")
-      backend.setValue("port", "3306");
+         .withUsername("root")
-      backend.setValue("databaseName", "opspath");
+         .withPassword("8BNWyoav8s79oi}Lqk") // todo - load securely
-      backend.setValue("username", "root");
+         .withName(MYSQL_BACKEND_NAME);
-      backend.setValue("password", "8BNWyoav8s79oi}Lqk"); // todo - load securely
+   }
-      return (backend);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static FilesystemBackendMetaData defineFilesystemBackend()
+   {
+      return new FilesystemBackendMetaData()
+         .withBasePath("/tmp/sample-filesystem")
+         .withName(FILESYSTEM_BACKEND_NAME);
    }
 
 
@@ -92,7 +125,7 @@ public class SampleMetaDataProvider
    {
       QTableMetaData table = new QTableMetaData();
       table.setName("carrier");
-      table.setBackendName(BACKEND_NAME);
+      table.setBackendName(MYSQL_BACKEND_NAME);
       table.setPrimaryKeyField("id");
 
       table.addField(new QFieldMetaData("id", QFieldType.INTEGER));
@@ -118,7 +151,7 @@ public class SampleMetaDataProvider
       return new QTableMetaData()
          .withName("person")
          .withLabel("Person")
-         .withBackendName(BACKEND_NAME)
+         .withBackendName(MYSQL_BACKEND_NAME)
          .withPrimaryKeyField("id")
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
@@ -128,4 +161,59 @@ public class SampleMetaDataProvider
          .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
          .withField(new QFieldMetaData("email", QFieldType.STRING));
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineTableCityFile()
+   {
+      return new QTableMetaData()
+         .withName("city")
+         .withLabel("Cities")
+         .withBackendName(FILESYSTEM_BACKEND_NAME)
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("name", QFieldType.STRING))
+         .withField(new QFieldMetaData("state", QFieldType.STRING)) // todo - state PVS.
+         .withBackendDetails(new FilesystemTableBackendDetails()
+            .withBasePath("cities")
+            .withCardinality(Cardinality.MANY)
+            .withRecordFormat(RecordFormat.CSV)
+         );
+   }
+
+
+
+   /*******************************************************************************
+    ** Define the 'greet people' process
+    *******************************************************************************/
+   private static QProcessMetaData defineProcessGreetPeople()
+   {
+      return new QProcessMetaData()
+         .withName(PROCESS_NAME_GREET)
+         .withLabel("Greet People")
+         .withTableName("person")
+         .addStep(new QBackendStepMetaData()
+            .withName("prepare")
+            .withCode(new QCodeReference()
+               .withName(MockBackendStep.class.getName())
+               .withCodeType(QCodeType.JAVA)
+               .withCodeUsage(QCodeUsage.FUNCTION)) // todo - needed, or implied in this context?
+            .withInputData(new QFunctionInputMetaData()
+               .withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
+               .withFieldList(List.of(
+                  new QFieldMetaData("greetingPrefix", QFieldType.STRING),
+                  new QFieldMetaData("greetingSuffix", QFieldType.STRING)
+               )))
+            .withOutputMetaData(new QFunctionOutputMetaData()
+               .withRecordListMetaData(new QRecordListMetaData()
+                  .withTableName("person")
+                  .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
+               )
+               .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
+         );
+   }
+
 }

src/main/java/com/kingsrook/sampleapp/customizers/FedExInvoiceCSVSyntaxFixer.java

@@ -0,0 +1,46 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.sampleapp.customizers;
+
+
+import java.util.function.Function;
+
+
+/*******************************************************************************
+ ** The fedex invoice files - they're CSV, but they have a handful of values
+ ** that we can't read, because they have an extra quote (") character within
+ ** a field.
+ **
+ ** It always looks like:  {"u
+ ** This function fixes it by stripping the " out of the middle, to just be: {u
+ *******************************************************************************/
+public class FedExInvoiceCSVSyntaxFixer implements Function<String, String>
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String apply(String s)
+   {
+      return (s.replaceAll("\\{\"u", "{u"));
+   }
+}

src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java

@@ -0,0 +1,126 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.sampleapp;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.actions.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.RunProcessAction;
+import com.kingsrook.qqq.backend.core.interfaces.mock.MockBackendStep;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+class SampleMetaDataProviderTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testCityFileTable() throws Exception
+   {
+      QTableMetaData fileTable       = SampleMetaDataProvider.defineTableCityFile();
+      File           destinationFile = copyTestFileToRandomNameUnderTable(fileTable);
+
+      try
+      {
+         QueryRequest queryRequest = new QueryRequest();
+         queryRequest.setInstance(SampleMetaDataProvider.defineInstance());
+         queryRequest.setTableName(fileTable.getName());
+
+         QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
+         System.out.println(queryResult);
+         Assertions.assertEquals(3, queryResult.getRecords().size(), "Should load all records from the file");
+      }
+      finally
+      {
+         destinationFile.delete();
+      }
+   }
+
+
+
+   private File copyTestFileToRandomNameUnderTable(QTableMetaData fedExTable) throws IOException
+   {
+      File destinationDir = new File(SampleMetaDataProvider.defineFilesystemBackend().getBasePath() + File.separator +
+         ((FilesystemTableBackendDetails) fedExTable.getBackendDetails()).getBasePath());
+      destinationDir.mkdirs();
+      File destinationFile = new File(destinationDir.getAbsolutePath() + File.separator + UUID.randomUUID());
+
+      FileUtils.writeStringToFile(destinationFile, """
+         id,name,state
+         1,Chester,IL
+         2,Red Bud,IL
+         3,Sparta,IL""");
+
+      return destinationFile;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testGreetProcess() throws Exception
+   {
+      QInstance         qInstance   = SampleMetaDataProvider.defineInstance();
+      QTableMetaData    personTable = SampleMetaDataProvider.defineTablePerson();
+      RunProcessRequest request     = new RunProcessRequest(qInstance);
+      request.setSession(new QSession());
+      request.setProcessName(SampleMetaDataProvider.PROCESS_NAME_GREET);
+
+      QueryRequest queryRequest = new QueryRequest(qInstance);
+      queryRequest.setTableName(personTable.getName());
+      queryRequest.setSession(new QSession());
+      QueryResult queryResult = new QueryAction().execute(queryRequest);
+
+      request.setRecords(queryResult.getRecords());
+      request.addValue(MockBackendStep.FIELD_GREETING_PREFIX, "Hello");
+      request.addValue(MockBackendStep.FIELD_GREETING_SUFFIX, "there");
+
+      RunProcessResult result = new RunProcessAction().execute(request);
+      assertNotNull(result);
+      assertNull(result.getError());
+      assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
+   }
+
+}