QQQ-26 add table exports

.circleci/config.yml

@@ -24,6 +24,8 @@ commands:
           name: Run Maven
           command: |
             mvn -s .circleci/mvn-settings.xml << parameters.maven_subcommand >>
+      - store_artifacts:
+          path: target/site/jacoco
       - run:
           name: Save test results
           command: |

.gitignore

@@ -28,3 +28,4 @@ target/
 
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
+.DS_Store

pom.xml

@@ -53,12 +53,12 @@
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-core</artifactId>
-         <version>0.2.0-SNAPSHOT</version>
+         <version>0.2.0-20220719.154219-3</version>
       </dependency>
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-module-rdbms</artifactId>
-         <version>0.2.0-SNAPSHOT</version>
+         <version>0.2.0-20220719.155012-5</version>
          <scope>test</scope>
       </dependency>
 
@@ -66,7 +66,7 @@
       <dependency>
          <groupId>io.javalin</groupId>
          <artifactId>javalin</artifactId>
-         <version>3.13.13</version>
+         <version>4.6.1</version>
       </dependency>
       <dependency>
          <groupId>com.konghq</groupId>
@@ -85,6 +85,12 @@
          <artifactId>slf4j-simple</artifactId>
          <version>1.7.36</version>
       </dependency>
+      <dependency>
+         <groupId>org.assertj</groupId>
+         <artifactId>assertj-core</artifactId>
+         <version>3.23.1</version>
+         <scope>test</scope>
+      </dependency>
 
       <!-- Common deps for all qqq modules -->
       <dependency>

src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java

@@ -24,14 +24,21 @@ package com.kingsrook.qqq.backend.javalin;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
 import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataAction;
 import com.kingsrook.qqq.backend.core.actions.metadata.ProcessMetaDataAction;
 import com.kingsrook.qqq.backend.core.actions.metadata.TableMetaDataAction;
+import com.kingsrook.qqq.backend.core.actions.reporting.ReportAction;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -49,6 +56,8 @@ import com.kingsrook.qqq.backend.core.model.actions.metadata.ProcessMetaDataInpu
 import com.kingsrook.qqq.backend.core.model.actions.metadata.ProcessMetaDataOutput;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataInput;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
@@ -182,36 +191,46 @@ public class QJavalinImplementation
    {
       return (() ->
       {
+         /////////////////////
+         // metadata routes //
+         /////////////////////
          path("/metaData", () ->
          {
             get("/", QJavalinImplementation::metaData);
-            path("/table/:table", () ->
+            path("/table/{table}", () ->
             {
                get("", QJavalinImplementation::tableMetaData);
             });
-            path("/process/:processName", () ->
+            path("/process/{processName}", () ->
             {
                get("", QJavalinImplementation::processMetaData);
             });
          });
-         path("/data", () ->
-         {
-            path("/:table", () ->
-            {
-               get("/", QJavalinImplementation::dataQuery);
-               post("/", QJavalinImplementation::dataInsert); // todo - internal to that method, if input is a list, do a bulk - else, single.
-               get("/count", QJavalinImplementation::dataCount);
 
-               // todo - add put and/or patch at this level (without a primaryKey) to do a bulk update based on primaryKeys in the records.
+         /////////////////////////
-               path("/:primaryKey", () ->
+         // table (data) routes //
-               {
+         /////////////////////////
-                  get("", QJavalinImplementation::dataGet);
+         path("/data/{table}", () ->
-                  patch("", QJavalinImplementation::dataUpdate);
+         {
-                  put("", QJavalinImplementation::dataUpdate); // todo - want different semantics??
+            get("/", QJavalinImplementation::dataQuery);
-                  delete("", QJavalinImplementation::dataDelete);
+            post("/", QJavalinImplementation::dataInsert); // todo - internal to that method, if input is a list, do a bulk - else, single.
-               });
+            get("/count", QJavalinImplementation::dataCount);
+            get("/export", QJavalinImplementation::dataExportWithoutFilename);
+            get("/export/{filename}", QJavalinImplementation::dataExportWithFilename);
+
+            // todo - add put and/or patch at this level (without a primaryKey) to do a bulk update based on primaryKeys in the records.
+            path("/{primaryKey}", () ->
+            {
+               get("", QJavalinImplementation::dataGet);
+               patch("", QJavalinImplementation::dataUpdate);
+               put("", QJavalinImplementation::dataUpdate); // todo - want different semantics??
+               delete("", QJavalinImplementation::dataDelete);
             });
          });
+
+         ////////////////////
+         // process routes //
+         ////////////////////
          path("", QJavalinProcessHandler.getRoutes());
       });
    }
@@ -221,16 +240,16 @@ public class QJavalinImplementation
    /*******************************************************************************
     **
     *******************************************************************************/
-   static void setupSession(Context context, AbstractActionInput request) throws QModuleDispatchException
+   static void setupSession(Context context, AbstractActionInput input) throws QModuleDispatchException
    {
       QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher();
-      QAuthenticationModuleInterface  authenticationModule            = qAuthenticationModuleDispatcher.getQModule(request.getAuthenticationMetaData());
+      QAuthenticationModuleInterface  authenticationModule            = qAuthenticationModuleDispatcher.getQModule(input.getAuthenticationMetaData());
 
       // todo - does this need some per-provider logic actually?  mmm...
       Map<String, String> authenticationContext = new HashMap<>();
       authenticationContext.put("sessionId", context.cookie("sessionId"));
       QSession session = authenticationModule.createSession(authenticationContext);
-      request.setSession(session);
+      input.setSession(session);
 
       context.cookie("sessionId", session.getIdReference(), SESSION_COOKIE_AGE);
    }
@@ -357,10 +376,10 @@ public class QJavalinImplementation
    {
       try
       {
-         String         tableName    = context.pathParam("table");
+         String         tableName  = context.pathParam("table");
-         QTableMetaData table        = qInstance.getTable(tableName);
+         QTableMetaData table      = qInstance.getTable(tableName);
-         String     primaryKey = context.pathParam("primaryKey");
+         String         primaryKey = context.pathParam("primaryKey");
-         QueryInput queryInput = new QueryInput(qInstance);
+         QueryInput     queryInput = new QueryInput(qInstance);
 
          setupSession(context, queryInput);
          queryInput.setTableName(tableName);
@@ -549,31 +568,181 @@ public class QJavalinImplementation
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void dataExportWithFilename(Context context)
+   {
+      String filename = context.pathParam("filename");
+      dataExport(context, Optional.of(filename));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void dataExportWithoutFilename(Context context)
+   {
+      dataExport(context, Optional.empty());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void dataExport(Context context, Optional<String> optionalFilename)
+   {
+      try
+      {
+         //////////////////////////////////////////
+         // read params from the request context //
+         //////////////////////////////////////////
+         String  tableName = context.pathParam("table");
+         String  format    = context.queryParam("format");
+         String  filter    = context.queryParam("filter");
+         Integer limit     = integerQueryParam(context, "limit");
+
+         /////////////////////////////////////////////////////////////////////////////////////////
+         // if a format query param wasn't given, then try to get file extension from file name //
+         /////////////////////////////////////////////////////////////////////////////////////////
+         if(!StringUtils.hasContent(format) && optionalFilename.isPresent() && StringUtils.hasContent(optionalFilename.get()))
+         {
+            String filename = optionalFilename.get();
+            if(filename.contains("."))
+            {
+               format = filename.substring(filename.lastIndexOf(".") + 1);
+            }
+         }
+
+         ReportFormat reportFormat;
+         try
+         {
+            reportFormat = ReportFormat.fromString(format);
+         }
+         catch(QUserFacingException e)
+         {
+            handleException(HttpStatus.Code.BAD_REQUEST, context, e);
+            return;
+         }
+
+         String filename = optionalFilename.orElse(tableName + "." + reportFormat.toString().toLowerCase(Locale.ROOT));
+
+         /////////////////////////////////////////////
+         // set up the report action's input object //
+         /////////////////////////////////////////////
+         ReportInput reportInput = new ReportInput(qInstance);
+         setupSession(context, reportInput);
+         reportInput.setTableName(tableName);
+         reportInput.setReportFormat(reportFormat);
+         reportInput.setFilename(filename);
+         reportInput.setLimit(limit);
+
+         String fields = stringQueryParam(context, "fields");
+         if(StringUtils.hasContent(fields))
+         {
+            reportInput.setFieldNames(List.of(fields.split(",")));
+         }
+
+         if(filter != null)
+         {
+            reportInput.setQueryFilter(JsonUtils.toObject(filter, QQueryFilter.class));
+         }
+
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         // set up the I/O pipe streams.                                                                      //
+         // Critically, we must NOT open the outputStream in a try-with-resources.  The thread that writes to //
+         // the stream must close it when it's done writing.                                                  //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         PipedOutputStream pipedOutputStream = new PipedOutputStream();
+         PipedInputStream  pipedInputStream  = new PipedInputStream();
+         pipedOutputStream.connect(pipedInputStream);
+         reportInput.setReportOutputStream(pipedOutputStream);
+
+         ReportAction reportAction = new ReportAction();
+         reportAction.preExecute(reportInput);
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         // start the async job.                                                                            //
+         // Critically, this must happen before the pipedInputStream is passed to the javalin result method //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         new AsyncJobManager().startJob("Javalin>ReportAction", (o) ->
+         {
+            try
+            {
+               reportAction.execute(reportInput);
+               return (true);
+            }
+            catch(Exception e)
+            {
+               pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes());
+               pipedOutputStream.close();
+               return (false);
+            }
+         });
+
+         ////////////////////////////////////////////
+         // set the response content type & stream //
+         ////////////////////////////////////////////
+         context.contentType(reportFormat.getMimeType());
+         context.header("Content-Disposition", "filename=" + filename);
+         context.result(pipedInputStream);
+
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         // we'd like to check to see if the job failed, and if so, to give the user an error...   //
+         // but if we "block" here, then piped streams seem to never flush, so we deadlock things. //
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         // AsyncJobStatus asyncJobStatus = asyncJobManager.waitForJob(jobUUID);
+         // if(asyncJobStatus.getState().equals(AsyncJobState.ERROR))
+         // {
+         //    System.out.println("Well, here we are...");
+         //    throw (new QUserFacingException("Error running report: " + asyncJobStatus.getCaughtException().getMessage()));
+         // }
+      }
+      catch(Exception e)
+      {
+         handleException(context, e);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    public static void handleException(Context context, Exception e)
+   {
+      handleException(null, context, e);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void handleException(HttpStatus.Code statusCode, Context context, Exception e)
    {
       QUserFacingException userFacingException = ExceptionUtils.findClassInRootChain(e, QUserFacingException.class);
       if(userFacingException != null)
       {
          if(userFacingException instanceof QNotFoundException)
          {
-            context.status(HttpStatus.NOT_FOUND_404)
+            int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.NOT_FOUND).getCode();
-               .result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
+            context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
          }
          else
          {
             LOG.info("User-facing exception", e);
-            context.status(HttpStatus.INTERNAL_SERVER_ERROR_500)
+            int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode();
-               .result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
+            context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
          }
       }
       else
       {
          LOG.warn("Exception in javalin request", e);
-         context.status(HttpStatus.INTERNAL_SERVER_ERROR_500)
+         int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode();
-            .result("{\"error\":\"" + e.getClass().getSimpleName() + " (" + e.getMessage() + ")\"}");
+         context.status(code).result("{\"error\":\"" + e.getClass().getSimpleName() + " (" + e.getMessage() + ")\"}");
       }
    }
 

src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java

@@ -87,15 +87,15 @@ public class QJavalinProcessHandler
       {
          path("/processes", () ->
          {
-            path("/:processName", () ->
+            path("/{processName}", () ->
             {
                get("/init", QJavalinProcessHandler::processInit);
                post("/init", QJavalinProcessHandler::processInit);
 
-               path("/:processUUID", () ->
+               path("/{processUUID}", () ->
                {
-                  post("/step/:step", QJavalinProcessHandler::processStep);
+                  post("/step/{step}", QJavalinProcessHandler::processStep);
-                  get("/status/:jobUUID", QJavalinProcessHandler::processStatus);
+                  get("/status/{jobUUID}", QJavalinProcessHandler::processStatus);
                   get("/records", QJavalinProcessHandler::processRecords);
                });
             });

src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java

@@ -27,6 +27,7 @@ import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import kong.unirest.HttpResponse;
 import kong.unirest.Unirest;
@@ -34,6 +35,7 @@ import org.eclipse.jetty.http.HttpStatus;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -52,8 +54,6 @@ class QJavalinImplementationTest extends QJavalinTestBase
 {
 
 
-
-
    /*******************************************************************************
     ** test the top-level meta-data endpoint
     **
@@ -269,7 +269,7 @@ class QJavalinImplementationTest extends QJavalinTestBase
    @Test
    public void test_dataQueryWithFilter()
    {
-      String               filterJson = "{\"criteria\":[{\"fieldName\":\"firstName\",\"operator\":\"EQUALS\",\"values\":[\"Tim\"]}]}";
+      String               filterJson = getFirstNameEqualsTimFilterJSON();
       HttpResponse<String> response   = Unirest.get(BASE_URL + "/data/person?filter=" + URLEncoder.encode(filterJson, StandardCharsets.UTF_8)).asString();
 
       assertEquals(200, response.getStatus());
@@ -284,6 +284,17 @@ class QJavalinImplementationTest extends QJavalinTestBase
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String getFirstNameEqualsTimFilterJSON()
+   {
+      return """
+         {"criteria":[{"fieldName":"firstName","operator":"EQUALS","values":["Tim"]}]}""";
+   }
+
+
+
    /*******************************************************************************
     ** test an insert
     **
@@ -375,4 +386,89 @@ class QJavalinImplementationTest extends QJavalinTestBase
       }));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExportCsvPerFileName()
+   {
+      HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/MyPersonExport.csv").asString();
+      assertEquals(200, response.getStatus());
+      assertEquals("text/csv", response.getHeaders().get("Content-Type").get(0));
+      assertEquals("filename=MyPersonExport.csv", response.getHeaders().get("Content-Disposition").get(0));
+      String[] csvLines = response.getBody().split("\n");
+      assertEquals(6, csvLines.length);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExportNoFormat()
+   {
+      HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/").asString();
+      assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus());
+      assertThat(response.getBody()).contains("Report format was not specified");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExportExcelPerFormatQueryParam()
+   {
+      HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/?format=xlsx").asString();
+      assertEquals(200, response.getStatus());
+      assertEquals(ReportFormat.XLSX.getMimeType(), response.getHeaders().get("Content-Type").get(0));
+      assertEquals("filename=person.xlsx", response.getHeaders().get("Content-Disposition").get(0));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExportFilterQueryParam()
+   {
+      String               filterJson = getFirstNameEqualsTimFilterJSON();
+      HttpResponse<String> response   = Unirest.get(BASE_URL + "/data/person/export/Favorite People.csv?filter=" + URLEncoder.encode(filterJson, StandardCharsets.UTF_8)).asString();
+      assertEquals("filename=Favorite People.csv", response.getHeaders().get("Content-Disposition").get(0));
+      String[] csvLines = response.getBody().split("\n");
+      assertEquals(2, csvLines.length);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExportFieldsQueryParam()
+   {
+      HttpResponse<String> response   = Unirest.get(BASE_URL + "/data/person/export/People.csv?fields=id,birthDate").asString();
+      String[] csvLines = response.getBody().split("\n");
+      assertEquals("""
+         "Id","Birth Date\"""", csvLines[0]);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExportSupportedFormat()
+   {
+      HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/?format=docx").asString();
+      assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus());
+      assertThat(response.getBody()).contains("Unsupported report format");
+   }
+
 }