mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-19 13:40:44 +00:00
Updated interface in sync processes; more status updates in ETL processes; Basepull only update timestamp if ran as basepull; javalin report endpoint;
This commit is contained in:
@ -184,6 +184,7 @@ public class QJavalinImplementation
|
||||
service = Javalin.create().start(port);
|
||||
service.routes(getRoutes());
|
||||
service.before(QJavalinImplementation::hotSwapQInstance);
|
||||
service.before((Context context) -> context.header("Content-Type", "application/json"));
|
||||
}
|
||||
|
||||
|
||||
@ -815,26 +816,9 @@ public class QJavalinImplementation
|
||||
String filter = context.queryParam("filter");
|
||||
Integer limit = integerQueryParam(context, "limit");
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if a format query param wasn't given, then try to get file extension from file name //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(!StringUtils.hasContent(format) && optionalFilename.isPresent() && StringUtils.hasContent(optionalFilename.get()))
|
||||
ReportFormat reportFormat = getReportFormat(context, optionalFilename, format);
|
||||
if(reportFormat == null)
|
||||
{
|
||||
String filename = optionalFilename.get();
|
||||
if(filename.contains("."))
|
||||
{
|
||||
format = filename.substring(filename.lastIndexOf(".") + 1);
|
||||
}
|
||||
}
|
||||
|
||||
ReportFormat reportFormat;
|
||||
try
|
||||
{
|
||||
reportFormat = ReportFormat.fromString(format);
|
||||
}
|
||||
catch(QUserFacingException e)
|
||||
{
|
||||
handleException(HttpStatus.Code.BAD_REQUEST, context, e);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -861,55 +845,21 @@ public class QJavalinImplementation
|
||||
exportInput.setQueryFilter(JsonUtils.toObject(filter, QQueryFilter.class));
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// set up the I/O pipe streams. //
|
||||
// Critically, we must NOT open the outputStream in a try-with-resources. The thread that writes to //
|
||||
// the stream must close it when it's done writing. //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
PipedOutputStream pipedOutputStream = new PipedOutputStream();
|
||||
PipedInputStream pipedInputStream = new PipedInputStream();
|
||||
pipedOutputStream.connect(pipedInputStream);
|
||||
exportInput.setReportOutputStream(pipedOutputStream);
|
||||
|
||||
ExportAction exportAction = new ExportAction();
|
||||
exportAction.preExecute(exportInput);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// start the async job. //
|
||||
// Critically, this must happen before the pipedInputStream is passed to the javalin result method //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
new AsyncJobManager().startJob("Javalin>ReportAction", (o) ->
|
||||
UnsafeFunction<PipedOutputStream, ExportAction> preAction = (PipedOutputStream pos) ->
|
||||
{
|
||||
try
|
||||
{
|
||||
exportAction.execute(exportInput);
|
||||
return (true);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes());
|
||||
pipedOutputStream.close();
|
||||
return (false);
|
||||
}
|
||||
});
|
||||
exportInput.setReportOutputStream(pos);
|
||||
|
||||
////////////////////////////////////////////
|
||||
// set the response content type & stream //
|
||||
////////////////////////////////////////////
|
||||
context.contentType(reportFormat.getMimeType());
|
||||
context.header("Content-Disposition", "filename=" + filename);
|
||||
context.result(pipedInputStream);
|
||||
ExportAction exportAction = new ExportAction();
|
||||
exportAction.preExecute(exportInput);
|
||||
return (exportAction);
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// we'd like to check to see if the job failed, and if so, to give the user an error... //
|
||||
// but if we "block" here, then piped streams seem to never flush, so we deadlock things. //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// AsyncJobStatus asyncJobStatus = asyncJobManager.waitForJob(jobUUID);
|
||||
// if(asyncJobStatus.getState().equals(AsyncJobState.ERROR))
|
||||
// {
|
||||
// System.out.println("Well, here we are...");
|
||||
// throw (new QUserFacingException("Error running report: " + asyncJobStatus.getCaughtException().getMessage()));
|
||||
// }
|
||||
UnsafeConsumer<ExportAction> execute = (ExportAction exportAction) ->
|
||||
{
|
||||
exportAction.execute(exportInput);
|
||||
};
|
||||
|
||||
runStreamedExportOrReport(context, reportFormat, filename, preAction, execute);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
@ -919,6 +869,122 @@ public class QJavalinImplementation
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@FunctionalInterface
|
||||
public interface UnsafeFunction<T, R>
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
R run(T t) throws Exception;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@FunctionalInterface
|
||||
public interface UnsafeConsumer<T>
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
void run(T t) throws Exception;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static <T> void runStreamedExportOrReport(Context context, ReportFormat reportFormat, String filename, UnsafeFunction<PipedOutputStream, T> preAction, UnsafeConsumer<T> executor) throws Exception
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// set up the I/O pipe streams. //
|
||||
// Critically, we must NOT open the outputStream in a try-with-resources. The thread that writes to //
|
||||
// the stream must close it when it's done writing. //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
PipedOutputStream pipedOutputStream = new PipedOutputStream();
|
||||
PipedInputStream pipedInputStream = new PipedInputStream();
|
||||
pipedOutputStream.connect(pipedInputStream);
|
||||
|
||||
T t = preAction.run(pipedOutputStream);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// start the async job. //
|
||||
// Critically, this must happen before the pipedInputStream is passed to the javalin result method //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
new AsyncJobManager().startJob("Javalin>ExportAction", (o) ->
|
||||
{
|
||||
try
|
||||
{
|
||||
executor.run(t);
|
||||
return (true);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes());
|
||||
pipedOutputStream.close();
|
||||
return (false);
|
||||
}
|
||||
});
|
||||
|
||||
////////////////////////////////////////////
|
||||
// set the response content type & stream //
|
||||
////////////////////////////////////////////
|
||||
context.contentType(reportFormat.getMimeType());
|
||||
context.header("Content-Disposition", "filename=" + filename);
|
||||
context.result(pipedInputStream);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// we'd like to check to see if the job failed, and if so, to give the user an error... //
|
||||
// but if we "block" here, then piped streams seem to never flush, so we deadlock things. //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// AsyncJobStatus asyncJobStatus = asyncJobManager.waitForJob(jobUUID);
|
||||
// if(asyncJobStatus.getState().equals(AsyncJobState.ERROR))
|
||||
// {
|
||||
// System.out.println("Well, here we are...");
|
||||
// throw (new QUserFacingException("Error running report: " + asyncJobStatus.getCaughtException().getMessage()));
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static ReportFormat getReportFormat(Context context, Optional<String> optionalFilename, String format)
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if a format query param wasn't given, then try to get file extension from file name //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(!StringUtils.hasContent(format) && optionalFilename.isPresent() && StringUtils.hasContent(optionalFilename.get()))
|
||||
{
|
||||
String filename = optionalFilename.get();
|
||||
if(filename.contains("."))
|
||||
{
|
||||
format = filename.substring(filename.lastIndexOf(".") + 1);
|
||||
}
|
||||
}
|
||||
|
||||
ReportFormat reportFormat;
|
||||
try
|
||||
{
|
||||
reportFormat = ReportFormat.fromString(format);
|
||||
}
|
||||
catch(QUserFacingException e)
|
||||
{
|
||||
handleException(HttpStatus.Code.BAD_REQUEST, context, e);
|
||||
return null;
|
||||
}
|
||||
return reportFormat;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -997,21 +1063,21 @@ public class QJavalinImplementation
|
||||
{
|
||||
if(userFacingException instanceof QNotFoundException)
|
||||
{
|
||||
int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.NOT_FOUND).getCode();
|
||||
context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
|
||||
statusCode = Objects.requireNonNullElse(statusCode, HttpStatus.Code.NOT_FOUND); // 404
|
||||
respondWithError(context, statusCode, userFacingException.getMessage());
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG.info("User-facing exception", e);
|
||||
int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode();
|
||||
context.status(code).result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
|
||||
statusCode = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR); // 500
|
||||
respondWithError(context, statusCode, userFacingException.getMessage());
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if(e instanceof QAuthenticationException)
|
||||
{
|
||||
context.status(HttpStatus.UNAUTHORIZED_401).result("{\"error\":\"" + e.getMessage() + "\"}");
|
||||
respondWithError(context, HttpStatus.Code.UNAUTHORIZED, e.getMessage()); // 401
|
||||
return;
|
||||
}
|
||||
|
||||
@ -1019,13 +1085,23 @@ public class QJavalinImplementation
|
||||
// default exception handling //
|
||||
////////////////////////////////
|
||||
LOG.warn("Exception in javalin request", e);
|
||||
int code = Objects.requireNonNullElse(statusCode, HttpStatus.Code.INTERNAL_SERVER_ERROR).getCode();
|
||||
context.status(code).result("{\"error\":\"" + e.getClass().getSimpleName() + " (" + e.getMessage() + ")\"}");
|
||||
respondWithError(context, HttpStatus.Code.INTERNAL_SERVER_ERROR, e.getClass().getSimpleName() + " (" + e.getMessage() + ")"); // 500
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static void respondWithError(Context context, HttpStatus.Code statusCode, String errorMessage)
|
||||
{
|
||||
context.status(statusCode.getCode());
|
||||
context.result(JsonUtils.toJson(Map.of("error", errorMessage)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Returns Integer if context has a valid int query parameter by the given name,
|
||||
** Returns null if no param (or empty value).
|
||||
|
||||
@ -27,6 +27,7 @@ import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PipedOutputStream;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDate;
|
||||
import java.util.ArrayList;
|
||||
@ -34,6 +35,7 @@ import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
@ -45,15 +47,19 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
|
||||
import com.kingsrook.qqq.backend.core.actions.async.JobGoingAsyncException;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
@ -62,6 +68,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.state.StateType;
|
||||
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
|
||||
@ -70,12 +77,14 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import io.javalin.apibuilder.EndpointGroup;
|
||||
import io.javalin.http.Context;
|
||||
import io.javalin.http.UploadedFile;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.eclipse.jetty.http.HttpStatus;
|
||||
import static io.javalin.apibuilder.ApiBuilder.get;
|
||||
import static io.javalin.apibuilder.ApiBuilder.path;
|
||||
import static io.javalin.apibuilder.ApiBuilder.post;
|
||||
@ -115,11 +124,131 @@ public class QJavalinProcessHandler
|
||||
});
|
||||
});
|
||||
get("/download/{file}", QJavalinProcessHandler::downloadFile);
|
||||
|
||||
path("/reports", () ->
|
||||
{
|
||||
path("/{reportName}", () ->
|
||||
{
|
||||
get("", QJavalinProcessHandler::reportWithoutFilename);
|
||||
get("/{fileName}", QJavalinProcessHandler::reportWithFilename);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void reportWithFilename(Context context)
|
||||
{
|
||||
String filename = context.pathParam("fileName");
|
||||
report(context, Optional.of(filename));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void reportWithoutFilename(Context context)
|
||||
{
|
||||
report(context, Optional.empty());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void report(Context context, Optional<String> optionalFilename)
|
||||
{
|
||||
try
|
||||
{
|
||||
//////////////////////////////////////////
|
||||
// read params from the request context //
|
||||
//////////////////////////////////////////
|
||||
String reportName = context.pathParam("reportName");
|
||||
String format = context.queryParam("format");
|
||||
|
||||
ReportFormat reportFormat = QJavalinImplementation.getReportFormat(context, optionalFilename, format);
|
||||
if(reportFormat == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
String filename = optionalFilename.orElse(reportName + "." + reportFormat.toString().toLowerCase(Locale.ROOT));
|
||||
|
||||
/////////////////////////////////////////////
|
||||
// set up the report action's input object //
|
||||
/////////////////////////////////////////////
|
||||
ReportInput reportInput = new ReportInput(QJavalinImplementation.qInstance);
|
||||
QJavalinImplementation.setupSession(context, reportInput);
|
||||
reportInput.setReportFormat(reportFormat);
|
||||
reportInput.setReportName(reportName);
|
||||
reportInput.setInputValues(null); // todo!
|
||||
reportInput.setFilename(filename);
|
||||
|
||||
QReportMetaData report = QJavalinImplementation.qInstance.getReport(reportName);
|
||||
if(report == null)
|
||||
{
|
||||
throw (new QNotFoundException("Report [" + reportName + "] is not found."));
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////
|
||||
// process the report's input fields, from the query string //
|
||||
//////////////////////////////////////////////////////////////
|
||||
for(QFieldMetaData inputField : CollectionUtils.nonNullList(report.getInputFields()))
|
||||
{
|
||||
try
|
||||
{
|
||||
boolean setValue = false;
|
||||
if(context.queryParamMap().containsKey(inputField.getName()))
|
||||
{
|
||||
String value = context.queryParamMap().get(inputField.getName()).get(0);
|
||||
Serializable typedValue = ValueUtils.getValueAsFieldType(inputField.getType(), value);
|
||||
reportInput.addInputValue(inputField.getName(), typedValue);
|
||||
setValue = true;
|
||||
}
|
||||
|
||||
if(inputField.getIsRequired() && !setValue)
|
||||
{
|
||||
QJavalinImplementation.respondWithError(context, HttpStatus.Code.BAD_REQUEST, "Missing query param value for required input field: [" + inputField.getName() + "]");
|
||||
return;
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
QJavalinImplementation.respondWithError(context, HttpStatus.Code.BAD_REQUEST, "Error processing query param [" + inputField.getName() + "]: " + e.getClass().getSimpleName() + " (" + e.getMessage() + ")");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
QJavalinImplementation.UnsafeFunction<PipedOutputStream, GenerateReportAction> preAction = (PipedOutputStream pos) ->
|
||||
{
|
||||
reportInput.setReportOutputStream(pos);
|
||||
|
||||
GenerateReportAction reportAction = new GenerateReportAction();
|
||||
// any pre-action?? export uses this for "too many rows" checks...
|
||||
return (reportAction);
|
||||
};
|
||||
|
||||
QJavalinImplementation.UnsafeConsumer<GenerateReportAction> execute = (GenerateReportAction generateReportAction) ->
|
||||
{
|
||||
generateReportAction.execute(reportInput);
|
||||
};
|
||||
|
||||
QJavalinImplementation.runStreamedExportOrReport(context, reportFormat, filename, preAction, execute);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
QJavalinImplementation.handleException(context, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
Reference in New Issue
Block a user