diff --git a/.circleci/config.yml b/.circleci/config.yml index fe4c371c..658f75d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,8 @@ commands: name: Run Maven command: | mvn -s .circleci/mvn-settings.xml << parameters.maven_subcommand >> + - store_artifacts: + path: target/site/jacoco - run: name: Save test results command: | diff --git a/.gitignore b/.gitignore index 39736a21..b65cb041 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ target/ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* +.DS_Store diff --git a/pom.xml b/pom.xml index 96c67854..81502b31 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ io.javalin javalin - 3.13.13 + 4.6.1 com.konghq @@ -85,6 +85,12 @@ slf4j-simple 1.7.36 + + org.assertj + assertj-core + 3.23.1 + test + diff --git a/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index aaf30649..e932fe4a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -24,14 +24,21 @@ package com.kingsrook.qqq.backend.javalin; import java.io.File; import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataAction; import com.kingsrook.qqq.backend.core.actions.metadata.ProcessMetaDataAction; import com.kingsrook.qqq.backend.core.actions.metadata.TableMetaDataAction; +import com.kingsrook.qqq.backend.core.actions.reporting.ReportAction; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; @@ -50,6 +57,8 @@ import com.kingsrook.qqq.backend.core.model.actions.metadata.ProcessMetaDataInpu import com.kingsrook.qqq.backend.core.model.actions.metadata.ProcessMetaDataOutput; import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; @@ -185,36 +194,46 @@ public class QJavalinImplementation { return (() -> { + ///////////////////// + // metadata routes // + ///////////////////// path("/metaData", () -> { get("/", QJavalinImplementation::metaData); - path("/table/:table", () -> + path("/table/{table}", () -> { get("", QJavalinImplementation::tableMetaData); }); - path("/process/:processName", () -> + path("/process/{processName}", () -> { get("", QJavalinImplementation::processMetaData); }); }); - path("/data", () -> - { - path("/:table", () -> - { - get("/", QJavalinImplementation::dataQuery); - post("/", QJavalinImplementation::dataInsert); // todo - internal to that method, if input is a list, do a bulk - else, single. - get("/count", QJavalinImplementation::dataCount); - // todo - add put and/or patch at this level (without a primaryKey) to do a bulk update based on primaryKeys in the records. - path("/:primaryKey", () -> - { - get("", QJavalinImplementation::dataGet); - patch("", QJavalinImplementation::dataUpdate); - put("", QJavalinImplementation::dataUpdate); // todo - want different semantics?? - delete("", QJavalinImplementation::dataDelete); - }); + ///////////////////////// + // table (data) routes // + ///////////////////////// + path("/data/{table}", () -> + { + get("/", QJavalinImplementation::dataQuery); + post("/", QJavalinImplementation::dataInsert); // todo - internal to that method, if input is a list, do a bulk - else, single. + get("/count", QJavalinImplementation::dataCount); + get("/export", QJavalinImplementation::dataExportWithoutFilename); + get("/export/{filename}", QJavalinImplementation::dataExportWithFilename); + + // todo - add put and/or patch at this level (without a primaryKey) to do a bulk update based on primaryKeys in the records. + path("/{primaryKey}", () -> + { + get("", QJavalinImplementation::dataGet); + patch("", QJavalinImplementation::dataUpdate); + put("", QJavalinImplementation::dataUpdate); // todo - want different semantics?? + delete("", QJavalinImplementation::dataDelete); }); }); + + //////////////////// + // process routes // + //////////////////// path("", QJavalinProcessHandler.getRoutes()); }); } @@ -224,17 +243,17 @@ public class QJavalinImplementation /******************************************************************************* ** *******************************************************************************/ - static void setupSession(Context context, AbstractActionInput request) throws QModuleDispatchException + static void setupSession(Context context, AbstractActionInput input) throws QModuleDispatchException { QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher(); - QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(request.getAuthenticationMetaData()); + QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(input.getAuthenticationMetaData()); try { Map authenticationContext = new HashMap<>(); authenticationContext.put(SESSION_ID_COOKIE_NAME, context.cookie(SESSION_ID_COOKIE_NAME)); QSession session = authenticationModule.createSession(qInstance, authenticationContext); - request.setSession(session); + input.setSession(session); context.cookie(SESSION_ID_COOKIE_NAME, session.getIdReference(), SESSION_COOKIE_AGE); } @@ -372,10 +391,10 @@ public class QJavalinImplementation { try { - String tableName = context.pathParam("table"); - QTableMetaData table = qInstance.getTable(tableName); - String primaryKey = context.pathParam("primaryKey"); - QueryInput queryInput = new QueryInput(qInstance); + String tableName = context.pathParam("table"); + QTableMetaData table = qInstance.getTable(tableName); + String primaryKey = context.pathParam("primaryKey"); + QueryInput queryInput = new QueryInput(qInstance); setupSession(context, queryInput); queryInput.setTableName(tableName); @@ -564,36 +583,187 @@ public class QJavalinImplementation + /******************************************************************************* + ** + *******************************************************************************/ + private static void dataExportWithFilename(Context context) + { + String filename = context.pathParam("filename"); + dataExport(context, Optional.of(filename)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void dataExportWithoutFilename(Context context) + { + dataExport(context, Optional.empty()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void dataExport(Context context, Optional optionalFilename) + { + try + { + ////////////////////////////////////////// + // read params from the request context // + ////////////////////////////////////////// + String tableName = context.pathParam("table"); + String format = context.queryParam("format"); + String filter = context.queryParam("filter"); + Integer limit = integerQueryParam(context, "limit"); + + ///////////////////////////////////////////////////////////////////////////////////////// + // if a format query param wasn't given, then try to get file extension from file name // + ///////////////////////////////////////////////////////////////////////////////////////// + if(!StringUtils.hasContent(format) && optionalFilename.isPresent() && StringUtils.hasContent(optionalFilename.get())) + { + String filename = optionalFilename.get(); + if(filename.contains(".")) + { + format = filename.substring(filename.lastIndexOf(".") + 1); + } + } + + ReportFormat reportFormat; + try + { + reportFormat = ReportFormat.fromString(format); + } + catch(QUserFacingException e) + { + handleException(HttpStatus.Code.BAD_REQUEST, context, e); + return; + } + + String filename = optionalFilename.orElse(tableName + "." + reportFormat.toString().toLowerCase(Locale.ROOT)); + + ///////////////////////////////////////////// + // set up the report action's input object // + ///////////////////////////////////////////// + ReportInput reportInput = new ReportInput(qInstance); + setupSession(context, reportInput); + reportInput.setTableName(tableName); + reportInput.setReportFormat(reportFormat); + reportInput.setFilename(filename); + reportInput.setLimit(limit); + + String fields = stringQueryParam(context, "fields"); + if(StringUtils.hasContent(fields)) + { + reportInput.setFieldNames(List.of(fields.split(","))); + } + + if(filter != null) + { + reportInput.setQueryFilter(JsonUtils.toObject(filter, QQueryFilter.class)); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up the I/O pipe streams. // + // Critically, we must NOT open the outputStream in a try-with-resources. The thread that writes to // + // the stream must close it when it's done writing. // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + PipedOutputStream pipedOutputStream = new PipedOutputStream(); + PipedInputStream pipedInputStream = new PipedInputStream(); + pipedOutputStream.connect(pipedInputStream); + reportInput.setReportOutputStream(pipedOutputStream); + + ReportAction reportAction = new ReportAction(); + reportAction.preExecute(reportInput); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // start the async job. // + // Critically, this must happen before the pipedInputStream is passed to the javalin result method // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + new AsyncJobManager().startJob("Javalin>ReportAction", (o) -> + { + try + { + reportAction.execute(reportInput); + return (true); + } + catch(Exception e) + { + pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes()); + pipedOutputStream.close(); + return (false); + } + }); + + //////////////////////////////////////////// + // set the response content type & stream // + //////////////////////////////////////////// + context.contentType(reportFormat.getMimeType()); + context.header("Content-Disposition", "filename=" + filename); + context.result(pipedInputStream); + + //////////////////////////////////////////////////////////////////////////////////////////// + // we'd like to check to see if the job failed, and if so, to give the user an error... // + // but if we "block" here, then piped streams seem to never flush, so we deadlock things. // + //////////////////////////////////////////////////////////////////////////////////////////// + // AsyncJobStatus asyncJobStatus = asyncJobManager.waitForJob(jobUUID); + // if(asyncJobStatus.getState().equals(AsyncJobState.ERROR)) + // { + // System.out.println("Well, here we are..."); + // throw (new QUserFacingException("Error running report: " + asyncJobStatus.getCaughtException().getMessage())); + // } + } + catch(Exception e) + { + handleException(context, e); + } + } + + + /******************************************************************************* ** *******************************************************************************/ public static void handleException(Context context, Exception e) + { + handleException(null, context, e); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void handleException(HttpStatus.Code statusCode, Context context, Exception e) { QUserFacingException userFacingException = ExceptionUtils.findClassInRootChain(e, QUserFacingException.class); if(userFacingException != null) { if(userFacingException instanceof QNotFoundException) { - context.status(HttpStatus.NOT_FOUND_404) - .result("{\"error\":\"" + userFacingException.getMessage() + "\"}"); - return; + int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.NOT_FOUND).getCode(); + context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}"); } else { LOG.info("User-facing exception", e); - context.status(HttpStatus.INTERNAL_SERVER_ERROR_500) - .result("{\"error\":\"" + userFacingException.getMessage() + "\"}"); - return; + int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode(); + context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}"); } } else { if(e instanceof QAuthenticationException) { - context.status(HttpStatus.UNAUTHORIZED_401) - .result("{\"error\":\"" + e.getMessage() + "\"}"); + context.status(HttpStatus.UNAUTHORIZED_401).result("{\"error\":\"" + e.getMessage() + "\"}"); return; } + + LOG.warn("Exception in javalin request", e); + int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode(); + context.status(code).result("{\"error\":\"" + e.getClass().getSimpleName() + " (" + e.getMessage() + ")\"}"); } //////////////////////////////// diff --git a/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 308634a9..1c0775e9 100644 --- a/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -87,15 +87,15 @@ public class QJavalinProcessHandler { path("/processes", () -> { - path("/:processName", () -> + path("/{processName}", () -> { get("/init", QJavalinProcessHandler::processInit); post("/init", QJavalinProcessHandler::processInit); - path("/:processUUID", () -> + path("/{processUUID}", () -> { - post("/step/:step", QJavalinProcessHandler::processStep); - get("/status/:jobUUID", QJavalinProcessHandler::processStatus); + post("/step/{step}", QJavalinProcessHandler::processStep); + get("/status/{jobUUID}", QJavalinProcessHandler::processStatus); get("/records", QJavalinProcessHandler::processRecords); }); }); diff --git a/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java index 459ff2bb..26604c8c 100644 --- a/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java @@ -27,6 +27,7 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import kong.unirest.HttpResponse; import kong.unirest.Unirest; @@ -34,6 +35,7 @@ import org.eclipse.jetty.http.HttpStatus; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -52,8 +54,6 @@ class QJavalinImplementationTest extends QJavalinTestBase { - - /******************************************************************************* ** test the top-level meta-data endpoint ** @@ -269,7 +269,7 @@ class QJavalinImplementationTest extends QJavalinTestBase @Test public void test_dataQueryWithFilter() { - String filterJson = "{\"criteria\":[{\"fieldName\":\"firstName\",\"operator\":\"EQUALS\",\"values\":[\"Tim\"]}]}"; + String filterJson = getFirstNameEqualsTimFilterJSON(); HttpResponse response = Unirest.get(BASE_URL + "/data/person?filter=" + URLEncoder.encode(filterJson, StandardCharsets.UTF_8)).asString(); assertEquals(200, response.getStatus()); @@ -284,6 +284,17 @@ class QJavalinImplementationTest extends QJavalinTestBase + /******************************************************************************* + ** + *******************************************************************************/ + private String getFirstNameEqualsTimFilterJSON() + { + return """ + {"criteria":[{"fieldName":"firstName","operator":"EQUALS","values":["Tim"]}]}"""; + } + + + /******************************************************************************* ** test an insert ** @@ -375,4 +386,89 @@ class QJavalinImplementationTest extends QJavalinTestBase })); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExportCsvPerFileName() + { + HttpResponse response = Unirest.get(BASE_URL + "/data/person/export/MyPersonExport.csv").asString(); + assertEquals(200, response.getStatus()); + assertEquals("text/csv", response.getHeaders().get("Content-Type").get(0)); + assertEquals("filename=MyPersonExport.csv", response.getHeaders().get("Content-Disposition").get(0)); + String[] csvLines = response.getBody().split("\n"); + assertEquals(6, csvLines.length); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExportNoFormat() + { + HttpResponse response = Unirest.get(BASE_URL + "/data/person/export/").asString(); + assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus()); + assertThat(response.getBody()).contains("Report format was not specified"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExportExcelPerFormatQueryParam() + { + HttpResponse response = Unirest.get(BASE_URL + "/data/person/export/?format=xlsx").asString(); + assertEquals(200, response.getStatus()); + assertEquals(ReportFormat.XLSX.getMimeType(), response.getHeaders().get("Content-Type").get(0)); + assertEquals("filename=person.xlsx", response.getHeaders().get("Content-Disposition").get(0)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExportFilterQueryParam() + { + String filterJson = getFirstNameEqualsTimFilterJSON(); + HttpResponse response = Unirest.get(BASE_URL + "/data/person/export/Favorite People.csv?filter=" + URLEncoder.encode(filterJson, StandardCharsets.UTF_8)).asString(); + assertEquals("filename=Favorite People.csv", response.getHeaders().get("Content-Disposition").get(0)); + String[] csvLines = response.getBody().split("\n"); + assertEquals(2, csvLines.length); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExportFieldsQueryParam() + { + HttpResponse response = Unirest.get(BASE_URL + "/data/person/export/People.csv?fields=id,birthDate").asString(); + String[] csvLines = response.getBody().split("\n"); + assertEquals(""" + "Id","Birth Date\"""", csvLines[0]); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExportSupportedFormat() + { + HttpResponse response = Unirest.get(BASE_URL + "/data/person/export/?format=docx").asString(); + assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus()); + assertThat(response.getBody()).contains("Unsupported report format"); + } + }