QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 21:20:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-26 add table exports

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-19 12:49:18 -05:00
parent be1482ce8f
commit caaf7e3d93
6 changed files with 314 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

229
src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -24,14 +24,21 @@ package com.kingsrook.qqq.backend.javalin;
import java.io.File;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataAction;
import com.kingsrook.qqq.backend.core.actions.metadata.ProcessMetaDataAction;
import com.kingsrook.qqq.backend.core.actions.metadata.TableMetaDataAction;
import com.kingsrook.qqq.backend.core.actions.reporting.ReportAction;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@ -49,6 +56,8 @@ import com.kingsrook.qqq.backend.core.model.actions.metadata.ProcessMetaDataInpu
import com.kingsrook.qqq.backend.core.model.actions.metadata.ProcessMetaDataOutput;
import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataInput;
import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataOutput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
@ -182,36 +191,46 @@ public class QJavalinImplementation
{
return (() ->
{
/////////////////////
// metadata routes //
/////////////////////
path("/metaData", () ->
{
get("/", QJavalinImplementation::metaData);
path("/table/:table", () ->
path("/table/{table}", () ->
{
get("", QJavalinImplementation::tableMetaData);
});
path("/process/:processName", () ->
path("/process/{processName}", () ->
{
get("", QJavalinImplementation::processMetaData);
});
});
path("/data", () ->
{
path("/:table", () ->
{
get("/", QJavalinImplementation::dataQuery);
post("/", QJavalinImplementation::dataInsert); // todo - internal to that method, if input is a list, do a bulk - else, single.
get("/count", QJavalinImplementation::dataCount);
// todo - add put and/or patch at this level (without a primaryKey) to do a bulk update based on primaryKeys in the records.
path("/:primaryKey", () ->
{
get("", QJavalinImplementation::dataGet);
patch("", QJavalinImplementation::dataUpdate);
put("", QJavalinImplementation::dataUpdate); // todo - want different semantics??
delete("", QJavalinImplementation::dataDelete);
});
/////////////////////////
// table (data) routes //
/////////////////////////
path("/data/{table}", () ->
{
get("/", QJavalinImplementation::dataQuery);
post("/", QJavalinImplementation::dataInsert); // todo - internal to that method, if input is a list, do a bulk - else, single.
get("/count", QJavalinImplementation::dataCount);
get("/export", QJavalinImplementation::dataExportWithoutFilename);
get("/export/{filename}", QJavalinImplementation::dataExportWithFilename);
// todo - add put and/or patch at this level (without a primaryKey) to do a bulk update based on primaryKeys in the records.
path("/{primaryKey}", () ->
{
get("", QJavalinImplementation::dataGet);
patch("", QJavalinImplementation::dataUpdate);
put("", QJavalinImplementation::dataUpdate); // todo - want different semantics??
delete("", QJavalinImplementation::dataDelete);
});
});
////////////////////
// process routes //
////////////////////
path("", QJavalinProcessHandler.getRoutes());
});
}
@ -221,16 +240,16 @@ public class QJavalinImplementation
/*******************************************************************************
**
*******************************************************************************/
static void setupSession(Context context, AbstractActionInput request) throws QModuleDispatchException
static void setupSession(Context context, AbstractActionInput input) throws QModuleDispatchException
{
QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher();
QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(request.getAuthenticationMetaData());
QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(input.getAuthenticationMetaData());
// todo - does this need some per-provider logic actually? mmm...
Map<String, String> authenticationContext = new HashMap<>();
authenticationContext.put("sessionId", context.cookie("sessionId"));
QSession session = authenticationModule.createSession(authenticationContext);
request.setSession(session);
input.setSession(session);
context.cookie("sessionId", session.getIdReference(), SESSION_COOKIE_AGE);
}
@ -357,10 +376,10 @@ public class QJavalinImplementation
{
try
{
String tableName = context.pathParam("table");
QTableMetaData table = qInstance.getTable(tableName);
String primaryKey = context.pathParam("primaryKey");
QueryInput queryInput = new QueryInput(qInstance);
String tableName = context.pathParam("table");
QTableMetaData table = qInstance.getTable(tableName);
String primaryKey = context.pathParam("primaryKey");
QueryInput queryInput = new QueryInput(qInstance);
setupSession(context, queryInput);
queryInput.setTableName(tableName);
@ -549,31 +568,181 @@ public class QJavalinImplementation
/*******************************************************************************
**
*******************************************************************************/
private static void dataExportWithFilename(Context context)
{
String filename = context.pathParam("filename");
dataExport(context, Optional.of(filename));
}
/*******************************************************************************
**
*******************************************************************************/
private static void dataExportWithoutFilename(Context context)
{
dataExport(context, Optional.empty());
}
/*******************************************************************************
**
*******************************************************************************/
private static void dataExport(Context context, Optional<String> optionalFilename)
{
try
{
//////////////////////////////////////////
// read params from the request context //
//////////////////////////////////////////
String tableName = context.pathParam("table");
String format = context.queryParam("format");
String filter = context.queryParam("filter");
Integer limit = integerQueryParam(context, "limit");
/////////////////////////////////////////////////////////////////////////////////////////
// if a format query param wasn't given, then try to get file extension from file name //
/////////////////////////////////////////////////////////////////////////////////////////
if(!StringUtils.hasContent(format) && optionalFilename.isPresent() && StringUtils.hasContent(optionalFilename.get()))
{
String filename = optionalFilename.get();
if(filename.contains("."))
{
format = filename.substring(filename.lastIndexOf(".") + 1);
}
}
ReportFormat reportFormat;
try
{
reportFormat = ReportFormat.fromString(format);
}
catch(QUserFacingException e)
{
handleException(HttpStatus.Code.BAD_REQUEST, context, e);
return;
}
String filename = optionalFilename.orElse(tableName + "." + reportFormat.toString().toLowerCase(Locale.ROOT));
/////////////////////////////////////////////
// set up the report action's input object //
/////////////////////////////////////////////
ReportInput reportInput = new ReportInput(qInstance);
setupSession(context, reportInput);
reportInput.setTableName(tableName);
reportInput.setReportFormat(reportFormat);
reportInput.setFilename(filename);
reportInput.setLimit(limit);
String fields = stringQueryParam(context, "fields");
if(StringUtils.hasContent(fields))
{
reportInput.setFieldNames(List.of(fields.split(",")));
}
if(filter != null)
{
reportInput.setQueryFilter(JsonUtils.toObject(filter, QQueryFilter.class));
}
///////////////////////////////////////////////////////////////////////////////////////////////////////
// set up the I/O pipe streams. //
// Critically, we must NOT open the outputStream in a try-with-resources. The thread that writes to //
// the stream must close it when it's done writing. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
PipedOutputStream pipedOutputStream = new PipedOutputStream();
PipedInputStream pipedInputStream = new PipedInputStream();
pipedOutputStream.connect(pipedInputStream);
reportInput.setReportOutputStream(pipedOutputStream);
ReportAction reportAction = new ReportAction();
reportAction.preExecute(reportInput);
/////////////////////////////////////////////////////////////////////////////////////////////////////
// start the async job. //
// Critically, this must happen before the pipedInputStream is passed to the javalin result method //
/////////////////////////////////////////////////////////////////////////////////////////////////////
new AsyncJobManager().startJob("Javalin>ReportAction", (o) ->
{
try
{
reportAction.execute(reportInput);
return (true);
}
catch(Exception e)
{
pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes());
pipedOutputStream.close();
return (false);
}
});
////////////////////////////////////////////
// set the response content type & stream //
////////////////////////////////////////////
context.contentType(reportFormat.getMimeType());
context.header("Content-Disposition", "filename=" + filename);
context.result(pipedInputStream);
////////////////////////////////////////////////////////////////////////////////////////////
// we'd like to check to see if the job failed, and if so, to give the user an error... //
// but if we "block" here, then piped streams seem to never flush, so we deadlock things. //
////////////////////////////////////////////////////////////////////////////////////////////
// AsyncJobStatus asyncJobStatus = asyncJobManager.waitForJob(jobUUID);
// if(asyncJobStatus.getState().equals(AsyncJobState.ERROR))
// {
// System.out.println("Well, here we are...");
// throw (new QUserFacingException("Error running report: " + asyncJobStatus.getCaughtException().getMessage()));
// }
}
catch(Exception e)
{
handleException(context, e);
}
}
/*******************************************************************************
**
*******************************************************************************/
public static void handleException(Context context, Exception e)
{
handleException(null, context, e);
}
/*******************************************************************************
**
*******************************************************************************/
public static void handleException(HttpStatus.Code statusCode, Context context, Exception e)
{
QUserFacingException userFacingException = ExceptionUtils.findClassInRootChain(e, QUserFacingException.class);
if(userFacingException != null)
{
if(userFacingException instanceof QNotFoundException)
{
context.status(HttpStatus.NOT_FOUND_404)
.result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.NOT_FOUND).getCode();
context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
}
else
{
LOG.info("User-facing exception", e);
context.status(HttpStatus.INTERNAL_SERVER_ERROR_500)
.result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode();
context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
}
}
else
{
LOG.warn("Exception in javalin request", e);
context.status(HttpStatus.INTERNAL_SERVER_ERROR_500)
.result("{\"error\":\"" + e.getClass().getSimpleName() + " (" + e.getMessage() + ")\"}");
int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode();
context.status(code).result("{\"error\":\"" + e.getClass().getSimpleName() + " (" + e.getMessage() + ")\"}");
}
}

8
src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java
View File

@ -87,15 +87,15 @@ public class QJavalinProcessHandler
{
path("/processes", () ->
{
path("/:processName", () ->
path("/{processName}", () ->
{
get("/init", QJavalinProcessHandler::processInit);
post("/init", QJavalinProcessHandler::processInit);
path("/:processUUID", () ->
path("/{processUUID}", () ->
{
post("/step/:step", QJavalinProcessHandler::processStep);
get("/status/:jobUUID", QJavalinProcessHandler::processStatus);
post("/step/{step}", QJavalinProcessHandler::processStep);
get("/status/{jobUUID}", QJavalinProcessHandler::processStatus);
get("/records", QJavalinProcessHandler::processRecords);
});
});

102
src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
View File

@ -27,6 +27,7 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import kong.unirest.HttpResponse;
import kong.unirest.Unirest;
@ -34,6 +35,7 @@ import org.eclipse.jetty.http.HttpStatus;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@ -52,8 +54,6 @@ class QJavalinImplementationTest extends QJavalinTestBase
{
/*******************************************************************************
** test the top-level meta-data endpoint
**
@ -269,7 +269,7 @@ class QJavalinImplementationTest extends QJavalinTestBase
@Test
public void test_dataQueryWithFilter()
{
String filterJson = "{\"criteria\":[{\"fieldName\":\"firstName\",\"operator\":\"EQUALS\",\"values\":[\"Tim\"]}]}";
String filterJson = getFirstNameEqualsTimFilterJSON();
HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person?filter=" + URLEncoder.encode(filterJson, StandardCharsets.UTF_8)).asString();
assertEquals(200, response.getStatus());
@ -284,6 +284,17 @@ class QJavalinImplementationTest extends QJavalinTestBase
/*******************************************************************************
**
*******************************************************************************/
private String getFirstNameEqualsTimFilterJSON()
{
return """
{"criteria":[{"fieldName":"firstName","operator":"EQUALS","values":["Tim"]}]}""";
}
/*******************************************************************************
** test an insert
**
@ -375,4 +386,89 @@ class QJavalinImplementationTest extends QJavalinTestBase
}));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testExportCsvPerFileName()
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/MyPersonExport.csv").asString();
assertEquals(200, response.getStatus());
assertEquals("text/csv", response.getHeaders().get("Content-Type").get(0));
assertEquals("filename=MyPersonExport.csv", response.getHeaders().get("Content-Disposition").get(0));
String[] csvLines = response.getBody().split("\n");
assertEquals(6, csvLines.length);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testExportNoFormat()
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/").asString();
assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus());
assertThat(response.getBody()).contains("Report format was not specified");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testExportExcelPerFormatQueryParam()
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/?format=xlsx").asString();
assertEquals(200, response.getStatus());
assertEquals(ReportFormat.XLSX.getMimeType(), response.getHeaders().get("Content-Type").get(0));
assertEquals("filename=person.xlsx", response.getHeaders().get("Content-Disposition").get(0));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testExportFilterQueryParam()
{
String filterJson = getFirstNameEqualsTimFilterJSON();
HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/Favorite People.csv?filter=" + URLEncoder.encode(filterJson, StandardCharsets.UTF_8)).asString();
assertEquals("filename=Favorite People.csv", response.getHeaders().get("Content-Disposition").get(0));
String[] csvLines = response.getBody().split("\n");
assertEquals(2, csvLines.length);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testExportFieldsQueryParam()
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/People.csv?fields=id,birthDate").asString();
String[] csvLines = response.getBody().split("\n");
assertEquals("""
"Id","Birth Date\"""", csvLines[0]);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testExportSupportedFormat()
{
HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/export/?format=docx").asString();
assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus());
assertThat(response.getBody()).contains("Unsupported report format");
}
}